Add http_Read() which reads from a socket but soaks up any prefeched

tail first and use it all the places where this logic was explicit
before.

Fix Refcounting on objects when we insert/deliver



git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@485 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache.h

@@ -260,6 +260,7 @@ int FetchHeaders(struct worker *w, struct sess *sp);
 /* cache_hash.c */
 struct object *HSH_Lookup(struct sess *sp);
 void HSH_Unbusy(struct object *o);
+void HSH_Ref(struct object *o);
 void HSH_Deref(struct object *o);
 void HSH_Init(void);
 
@@ -270,6 +271,7 @@ int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char *
 int http_GetStatus(struct http *hp);
 int http_HdrIs(struct http *hp, const char *hdr, const char *val);
 int http_GetTail(struct http *hp, unsigned len, char **b, char **e);
+int http_Read(struct http *hp, int fd, char *b, unsigned len);
 void http_RecvHead(struct http *hp, int fd, struct event_base *eb, http_callback_f *func, void *arg);
 int http_DissectRequest(struct http *sp, int fd);
 int http_DissectResponse(struct http *sp, int fd);

bin/varnishd/cache_center.c

@@ -62,6 +62,8 @@ cnt_deliver(struct sess *sp)
 {
 
 	vca_write_obj(sp->wrk, sp);
+	HSH_Deref(sp->obj);
+	sp->obj = NULL;
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -82,6 +84,7 @@ cnt_done(struct sess *sp)
 {
 	char *b;
 
+	assert(sp->obj == NULL);
 	if (http_GetHdr(sp->http, "Connection", &b) &&
 	    !strcmp(b, "close")) {
 		vca_close_session(sp, "Connection header");
@@ -182,6 +185,7 @@ cnt_fetch(struct sess *sp)
 		sp->obj->pass = 1;
 		sp->obj->cacheable = 1;
 		HSH_Unbusy(sp->obj);
+		/* Don't HSH_Deref(sp->obj); we need the ref for storage */
 		sp->obj = NULL;
 		sp->step = STP_PASSBODY;
 		return (0);
@@ -189,6 +193,8 @@ cnt_fetch(struct sess *sp)
 	if (sp->handling == VCL_RET_INSERT) {
 		sp->obj->cacheable = 1;
 		FetchBody(sp->wrk, sp);
+		HSH_Ref(sp->obj); /* get another, STP_DELIVER will deref */
+		HSH_Unbusy(sp->obj);
 		sp->step = STP_DELIVER;
 		return (0);
 	}
@@ -510,6 +516,8 @@ cnt_recv(struct sess *sp)
 	sp->vcl = VCL_Get();
 	SES_RefSrcAddr(sp);
 
+	assert(sp->obj == NULL);
+
 	done = http_DissectRequest(sp->http, sp->fd);
 	if (done != 0) {
 		RES_Error(sp->wrk, sp, done, NULL);

bin/varnishd/cache_fetch.c

@@ -33,7 +33,6 @@ static int
 fetch_straight(const struct sess *sp, int fd, struct http *hp, char *b)
 {
 	int i;
-	char *e;
 	unsigned char *p;
 	off_t	cl;
 	struct storage *st;
@@ -50,15 +49,8 @@ fetch_straight(const struct sess *sp, int fd, struct http *hp, char *b)
 	i &= ~O_NONBLOCK;
 	i = fcntl(fd, F_SETFL, i);
 
-	if (http_GetTail(hp, cl, &b, &e)) {
-		i = e - b;
-		memcpy(p, b, i);
-		p += i;
-		cl -= i;
-	}
-
 	while (cl != 0) {
-		i = read(fd, p, cl);
+		i = http_Read(hp, fd, p, cl);
 		assert(i > 0);	/* XXX seen */
 		p += i;
 		cl -= i;
@@ -73,7 +65,7 @@ static int
 fetch_chunked(const struct sess *sp, int fd, struct http *hp)
 {
 	int i;
-	char *b, *q, *e;
+	char *q;
 	unsigned char *p;
 	struct storage *st;
 	unsigned u, v;
@@ -88,16 +80,10 @@ fetch_chunked(const struct sess *sp, int fd, struct http *hp)
 	bp = buf;
 	st = NULL;
 	while (1) {
-		if (http_GetTail(hp, be - bp, &b, &e)) {
-			memcpy(bp, b, e - b);
-			bp += e - b;
-			*bp = '\0';
-		} else {
-			i = read(fd, bp, be - bp);
+		i = http_Read(hp, fd, bp, be - bp);
 		assert(i >= 0);
 		bp += i;
 		*bp = '\0';
-		}
 		u = strtoul(buf, &q, 16);
 		if (q == NULL || q == buf)
 			continue;
@@ -151,15 +137,8 @@ fetch_chunked(const struct sess *sp, int fd, struct http *hp)
 				break;
 			if (v == 0)
 				continue;
-			if (http_GetTail(hp, v, &b, &e)) {
-				memcpy(p, b, e - b);
-				p += e - b;
-				st->len += e - b;
-				v -= e - b;
-				u -= e - b;
-			}
 			while (v > 0) {
-				i = read(fd, p, v);
+				i = http_Read(hp, fd, p, v);
 				assert(i > 0);
 				st->len += i;
 				v -= i;
@@ -183,7 +162,6 @@ static int
 fetch_eof(const struct sess *sp, int fd, struct http *hp)
 {
 	int i;
-	char *b, *e;
 	unsigned char *p;
 	struct storage *st;
 	unsigned v;
@@ -204,15 +182,7 @@ fetch_eof(const struct sess *sp, int fd, struct http *hp)
 		}
 		assert(p != NULL);
 		assert(st != NULL);
-		if (http_GetTail(hp, v, &b, &e)) {
-			memcpy(p, b, e - b);
-			p += e - b;
-			v -= e - b;
-			st->len += e - b;
-			sp->obj->len += e - b;
-			*p = '\0';
-		}
-		i = read(fd, p, v);
+		i = http_Read(hp, fd, p, v);
 		assert(i >= 0);
 		if (i == 0)
 		     break;

bin/varnishd/cache_hash.c

@@ -143,6 +143,17 @@ HSH_Unbusy(struct object *o)
 	}
 }
 
+void
+HSH_Ref(struct object *o)
+{
+	struct objhead *oh;
+
+	oh = o->objhead;
+	AZ(pthread_mutex_lock(&oh->mtx));
+	o->refcnt++;
+	AZ(pthread_mutex_unlock(&oh->mtx));
+}
+
 void
 HSH_Deref(struct object *o)
 {

bin/varnishd/cache_http.c

@@ -120,6 +120,33 @@ http_GetTail(struct http *hp, unsigned len, char **b, char **e)
 	return (1);
 }
 
+/* Read from fd, but soak up any tail first */
+
+int
+http_Read(struct http *hp, int fd, char *b, unsigned len)
+{
+	int i;
+	unsigned u;
+
+	u = 0;
+	if (hp->t < hp->v) {
+		u = hp->v - hp->t;
+		if (u > len)
+			u = len;
+		memcpy(b, hp->t, u);
+		hp->t += u;
+		b += u;
+		len -= u;
+	}
+	if (len > 0) {
+		i = read(fd, b, len);
+		if (i < 0)
+			return (i);
+		u += i;
+	}
+	return (u);
+}
+
 int
 http_GetStatus(struct http *hp)
 {

bin/varnishd/cache_pass.c

@@ -25,7 +25,6 @@ static int
 pass_straight(struct sess *sp, int fd, struct http *hp, char *bi)
 {
 	int i;
-	char *b, *e;
 	off_t	cl;
 	unsigned c;
 	char buf[PASS_BUFSIZ];
@@ -43,13 +42,7 @@ pass_straight(struct sess *sp, int fd, struct http *hp, char *bi)
 		c = cl;
 		if (c > sizeof buf)
 			c = sizeof buf;
-		if (http_GetTail(hp, c, &b, &e)) {
-			i = e - b;
-			vca_write(sp, b, i);
-			cl -= i;
-			continue;
-		}
-		i = read(fd, buf, c);
+		i = http_Read(hp, fd, buf, c);
 		if (i == 0 && bi == NULL)
 			return (1);
 		assert(i > 0);
@@ -81,15 +74,10 @@ pass_chunked(struct sess *sp, int fd, struct http *hp)
 	be = buf + sizeof buf;
 	p = buf;
 	while (1) {
-		if (http_GetTail(hp, be - bp, &b, &e)) {
-			memcpy(bp, b, e - b);
-			bp += e - b;
-		} else {
-			/* XXX: must be safe from backend */
+		i = http_Read(hp, fd, bp, be - bp);
 		i = read(fd, bp, be - bp);
 		assert(i > 0);
 		bp += i;
-		}
 		/* buffer valid from p to bp */
 
 		u = strtoul(p, &q, 16);
@@ -159,7 +147,7 @@ PassBody(struct worker *w, struct sess *sp)
 	vc = sp->vbc;
 	assert(vc != NULL);
 
-	http_BuildSbuf(sp->fd, Build_Reply, w->sb, hp);
+	http_BuildSbuf(sp->fd, Build_Passreply, w->sb, hp);
 	sbuf_cat(w->sb, "\r\n");
 	sbuf_finish(w->sb);
 	vca_write(sp, sbuf_data(w->sb), sbuf_len(w->sb));

bin/varnishd/cache_pipe.c

@@ -59,7 +59,7 @@ PipeSession(struct worker *w, struct sess *sp)
 	http_BuildSbuf(vc->fd, Build_Pipe, w->sb, sp->http);
 	i = write(vc->fd, sbuf_data(w->sb), sbuf_len(w->sb));
 	assert(i == sbuf_len(w->sb));
-	if (http_GetTail(sp->http, 0, &b, &e) && b != e) { /* XXX */
+	if (http_GetTail(sp->http, 0, &b, &e) && b != e) {
 		i = write(vc->fd, b, e - b);
 		if (i != e - b) {
 			close (vc->fd);