Initial http_GetHdrField() function.

Improve chunked encoding, allocate big storage chunks and trim the
last one at the end, instead of one storage chunk for each chunk
the remote server sends.

Call RFC2616 policy code.

Store headers from backend in cache and return to client.



git-svn-id: http://www.varnish-cache.org/svn/trunk@194 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache.h

@@ -41,6 +41,7 @@ struct storage {
 	TAILQ_ENTRY(storage)	list;
 	unsigned char		*ptr;
 	unsigned		len;
+	unsigned		space;
 	void			*priv;
 	struct stevedore	*stevedore;
 };
@@ -81,6 +82,7 @@ struct http;
 struct http *http_New(void);
 void http_Delete(struct http *hp);
 int http_GetHdr(struct http *hp, const char *hdr, char **ptr);
+int http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr);
 int http_GetStatus(struct http *hp);
 int http_HdrIs(struct http *hp, const char *hdr, const char *val);
 int http_GetTail(struct http *hp, unsigned len, char **b, char **e);
@@ -124,3 +126,7 @@ cli_func_t	cli_func_config_load;
 cli_func_t	cli_func_config_unload;
 cli_func_t	cli_func_config_use;
 #endif
+
+/* rfc2616.c */
+void RFC2616_Age(struct http *hp, time_t, time_t);
+

bin/varnishd/cache_fetch.c

@@ -4,6 +4,7 @@
 
 #include <assert.h>
 #include <stdio.h>
+#include <ctype.h>
 #include <inttypes.h>
 #include <unistd.h>
 #include <string.h>
@@ -22,6 +23,17 @@
 #include "vcl_lang.h"
 #include "cache.h"
 
+/*
+ * Chunked encoding is a hack.  We prefer to have a single or a few
+ * large storage objects, and a terribly long list of small ones.
+ * If our stevedore can trim, we alloc big chunks and trim the last one
+ * at the end know the result.
+ *
+ * Good testcase: http://www.washingtonpost.com/
+ */
+#define CHUNK_PREALLOC		(128 * 1024)
+
+/*--------------------------------------------------------------------*/
 static int
 fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char *b)
 {
@@ -45,7 +57,6 @@ fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char
 
 	if (http_GetTail(hp, cl, &b, &e)) {
 		i = e - b;
-		VSL(SLT_Debug, 0, "Fetch_Tail %jd %d", cl, i);
 		memcpy(p, b, i);
 		p += i;
 		cl -= i;
@@ -53,7 +64,6 @@ fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char
 
 	while (cl != 0) {
 		i = read(fd, p, cl);
-		VSL(SLT_Debug, 0, "Fetch_Read %jd %d", cl, i);
 		assert(i > 0);
 		p += i;
 		cl -= i;
@@ -68,6 +78,9 @@ fetch_straight(struct worker *w, struct sess *sp, int fd, struct http *hp, char
 
 }
 
+/*--------------------------------------------------------------------*/
+/* XXX: Cleanup.  It must be possible somehow :-( */
+
 static int
 fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp)
 {
@@ -75,7 +88,7 @@ fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp)
 	char *b, *q, *e;
 	unsigned char *p;
 	struct storage *st;
-	unsigned u;
+	unsigned u, v;
 	char buf[20];
 	char *bp, *be;
 
@@ -84,46 +97,92 @@ fetch_chunked(struct worker *w, struct sess *sp, int fd, struct http *hp)
 	i = fcntl(fd, F_SETFL, i);
 
 	be = buf + sizeof buf;
+	bp = buf;
+	st = NULL;
 	while (1) {
-		bp = buf;
 		if (http_GetTail(hp, be - bp, &b, &e)) {
 			memcpy(bp, b, e - b);
 			bp += e - b;
+			*bp = '\0';
 		} else {
 			i = read(fd, bp, be - bp);
 			assert(i >= 0);
 			bp += i;
+			*bp = '\0';
 		}
 		u = strtoul(buf, &q, 16);
-		if (q == NULL || (*q != '\n' && *q != '\r'))
+		if (q == NULL || q == buf)
 			continue;
+		assert(isspace(*q));
+		while (*q == '\t' || *q == ' ')
+			q++;
 		if (*q == '\r')
 			q++;
 		assert(*q == '\n');
 		q++;
 		if (u == 0)
 			break;
-		st = stevedore->alloc(stevedore, u);
-		TAILQ_INSERT_TAIL(&sp->obj->store, st, list);
-		st->len = u;
 		sp->obj->len += u;
-		p = st->ptr;
-		memcpy(p, q, bp - q);
-		p += bp - q;
-		u -= bp - q;
-		if (http_GetTail(hp, u, &b, &e)) {
-			memcpy(p, b, e - b);
-			p += e - b;
-			u -= e - b;
-		}
+
 		while (u > 0) {
-			i = read(fd, p, u);
-			assert(i > 0);
-			u -= i;
-			p += i;
+			if (st != NULL && st->len < st->space) {
+				p = st->ptr + st->len;
+			} else {
+				st = stevedore->alloc(stevedore,
+				    stevedore->trim == NULL ? u : CHUNK_PREALLOC);
+				TAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+				p = st->ptr;
+			}
+			v = st->space - st->len;
+			if (v > u)
+				v = u;
+
+			i = bp - q;
+			if (i == 0) {
+			} else if (v > i) {
+				memcpy(p, q, i);
+				p += i;
+				st->len += i;
+				u -= i;
+				v -= i;
+				q = bp = buf;
+			} else if (i >= v) {
+				memcpy(p, q, v);
+				p += v;
+				st->len += i;
+				q += v;
+				u -= v;
+				v -= v;
+				if (u == 0 && bp > q) {
+					memcpy(buf, q, bp - q);
+					q = bp = buf + (bp - q);
+				}
+			}
+			if (u == 0)
+				break;
+			if (v == 0)
+				continue;
+			if (http_GetTail(hp, v, &b, &e)) {
+				memcpy(p, b, e - b);
+				p += e - b;
+				st->len += e - b;
+				v -= e - b;
+				u -= e - b;
+			}
+			while (v > 0) {
+				i = read(fd, p, v);
+				assert(i > 0);
+				st->len += i;
+				v -= i;
+				u -= i;
+				p += i;
+			}
 		}
 	}
 
+	if (st != NULL && stevedore->trim != NULL)
+		stevedore->trim(st, st->len);
+
 	http_BuildSbuf(2, w->sb, hp);
 
 	vca_write_obj(sp, w->sb);
@@ -141,6 +200,7 @@ FetchSession(struct worker *w, struct sess *sp)
 	void *fd_token;
 	struct http *hp;
 	char *b;
+	time_t t_req, t_resp;
 
 	fd = VBE_GetFd(sp->backend, &fd_token);
 	assert(fd != -1);
@@ -150,6 +210,7 @@ FetchSession(struct worker *w, struct sess *sp)
 	http_BuildSbuf(1, w->sb, sp->http);
 	i = write(fd, sbuf_data(w->sb), sbuf_len(w->sb));
 	assert(i == sbuf_len(w->sb));
+	time(&t_req);
 
 	/* XXX: copy any contents */
 
@@ -159,15 +220,21 @@ FetchSession(struct worker *w, struct sess *sp)
 	 */
 	http_RecvHead(hp, fd, w->eb, NULL, NULL);
 	event_base_loop(w->eb, 0);
+	time(&t_resp);
 	http_Dissect(hp, fd, 2);
 
+	RFC2616_Age(hp, t_req, t_resp);
+
 	switch (http_GetStatus(hp)) {
 	case 200:
+	case 301:
+		http_BuildSbuf(3, w->sb, hp);
 		/* XXX: fill in object from headers */
 		sp->obj->valid = 1;
 		sp->obj->cacheable = 1;
+		sp->obj->header = strdup(sbuf_data(w->sb));
 		break;
-	case 301:
+	case 391:
 		sp->obj->valid = 0;
 		sp->obj->cacheable = 0;
 		break;

bin/varnishd/cache_http.c

@@ -101,6 +101,39 @@ http_GetHdr(struct http *hp, const char *hdr, char **ptr)
 	return (0);
 }
 
+int
+http_GetHdrField(struct http *hp, const char *hdr, const char *field, char **ptr)
+{
+	char *h;
+	int fl;
+
+	if (!http_GetHdr(hp, hdr, &h))
+		return (0);
+	fl = strlen(field);
+	while (*h) {
+		if (isspace(*h)) {
+			h++;
+			continue;
+		}
+		if (*h == ',') {
+			h++;
+			continue;
+		}
+		if (memcmp(h, field, fl) ||
+		    isalpha(h[fl]) || h[fl] == '-') {
+			while (*h && !(isspace(*h) || *h == ','))
+				h++;
+			continue;
+		}
+		if (h[fl] == '=')
+			*ptr = &h[fl + 1];
+		else
+			*ptr = NULL;
+		return (1);
+	}
+	return (0);
+}
+
 int
 http_HdrIs(struct http *hp, const char *hdr, const char *val)
 {
@@ -356,7 +389,7 @@ http_BuildSbuf(int resp, struct sbuf *sb, struct http *hp)
 
 	sbuf_clear(sb);
 	assert(sb != NULL);
-	if (resp == 2) {
+	if (resp == 2 || resp == 3) {
 		sbuf_cat(sb, hp->proto);
 		sbuf_cat(sb, " ");
 		sbuf_cat(sb, hp->status);
@@ -382,6 +415,7 @@ http_BuildSbuf(int resp, struct sbuf *sb, struct http *hp)
 		sbuf_cat(sb, hp->hdr[u]);
 		sbuf_cat(sb, "\r\n");
 	}
-	sbuf_cat(sb, "\r\n");
+	if (resp != 3)
+		sbuf_cat(sb, "\r\n");
 	sbuf_finish(sb);
 }

bin/varnishd/cache_pool.c

@@ -71,10 +71,9 @@ DeliverSession(struct worker *w, struct sess *sp)
 
 	sbuf_clear(w->sb);
 	sbuf_printf(w->sb,
-	    "HTTP/1.1 200 OK\r\n"
-	    "Server: Varnish\r\n"
+	    "%sServer: Varnish\r\n"
 	    "Content-Length: %u\r\n"
-	    "\r\n", sp->obj->len);
+	    "\r\n", sp->obj->header, sp->obj->len);
 
 	vca_write_obj(sp, w->sb);
 	return (1);