Originally, we did not have the bereq.* and beresp.* variable sets,

and instead (ab)used the obj.* set.

This allowed, amongst other things, the headers of cached objects
to be munged by subsequent cache-hits.

This was just as bad an idea as it sounds.

The downside was, that we had to reserve object workspace for the
potentially created headers, and this really bites, memory wise,
if your content is many small objects.

This commit allows, and sets obj_workspace to zero, allocating only
as much space as is necessary for the actual headers after vcl_fetch()
has returned.

The downside is, without extra workspace, you can no longer change
the content of obj.* string variables after the vcl_fetch() which
instantiated the object.

For now we still allow the obj_workspace to be set to reserve such
space, but in all likelyhood, this ability will disappear in the
future.





git-svn-id: http://www.varnish-cache.org/svn/trunk/varnish-cache@4224 d4fa192b-c00b-0410-8231-f00ffab90ce4

bin/varnishd/cache.h

@@ -154,6 +154,7 @@ struct http {
 	txt			hd[HTTP_HDR_MAX];
 	unsigned char		hdf[HTTP_HDR_MAX];
 #define HDF_FILTER		(1 << 0)	/* Filtered by Connection */
+#define HDF_COPY		(1 << 1)	/* Copy this field */
 	unsigned		nhd;
 };
 
@@ -512,6 +513,7 @@ void Fetch_Init(void);
 
 /* cache_http.c */
 const char *http_StatusMessage(unsigned);
+unsigned http_EstimateWS(struct http *fm, unsigned how);
 void HTTP_Init(void);
 void http_ClrHeader(struct http *to);
 unsigned http_Write(struct worker *w, const struct http *hp, int resp);

bin/varnishd/cache_center.c

@@ -334,7 +334,7 @@ cnt_error(struct sess *sp)
 	if (sp->obj == NULL) {
 		HSH_Prealloc(sp);
 		sp->wrk->cacheable = 0;
-		sp->obj = HSH_NewObject(sp);
+		sp->obj = HSH_NewObject(sp, 0);
 		sp->obj->xid = sp->xid;
 		sp->obj->entered = sp->t_req;
 	} else {
@@ -415,7 +415,7 @@ cnt_fetch(struct sess *sp)
 	int i;
 	struct http *hp, *hp2;
 	char *b;
-	unsigned handling;
+	unsigned handling, l;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
@@ -516,11 +516,16 @@ cnt_fetch(struct sess *sp)
 		AZ(sp->objcore);
 	}
 
+	l = http_EstimateWS(sp->wrk->beresp, HTTPH_A_INS);
+
+	/* Space for producing a Content-Length: header */
+	l += 30;
+
 	/*
 	 * XXX: If we have a Length: header, we should allocate the body
 	 * XXX: also.
  	 */
-	sp->obj = HSH_NewObject(sp);
+	sp->obj = HSH_NewObject(sp, l);
 
 	if (sp->objhead != NULL) {
 		CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);

bin/varnishd/cache_hash.c

@@ -82,22 +82,27 @@ HSH_Grace(double g)
 }
 
 struct object *
-HSH_NewObject(struct sess *sp)
+HSH_NewObject(struct sess *sp, unsigned l)
 {
 	struct object *o;
 	struct storage *st;
 	void *p;
 
+	if (l == 0)
+		l = 1024;
+	if (params->obj_workspace > 0 && params->obj_workspace > l)
+		l =  params->obj_workspace;
+
 	if (!sp->wrk->cacheable) {
-		p = malloc(sizeof *o + params->obj_workspace);
+		p = malloc(sizeof *o + l);
 		XXXAN(p);
 		o = p;
 		p = o + 1;
 		memset(o, 0, sizeof *o);
 		o->magic = OBJECT_MAGIC;
-		WS_Init(o->ws_o, "obj", p, params->obj_workspace);
+		WS_Init(o->ws_o, "obj", p, l);
 	} else {
-		st = STV_alloc(sp, params->obj_workspace);
+		st = STV_alloc(sp, sizeof *o + l);
 		XXXAN(st);
 		assert(st->space > sizeof *o);
 		o = (void *)st->ptr; /* XXX: align ? */

bin/varnishd/cache_http.c

@@ -624,6 +624,34 @@ http_copyheader(struct worker *w, int fd, struct http *to,
 	}
 }
 
+/*--------------------------------------------------------------------
+ * Estimate how much workspace we need to Filter this header according
+ * to 'how'.
+ */
+
+unsigned 
+http_EstimateWS(struct http *fm, unsigned how)
+{
+	unsigned u, l;
+
+	l = 0;
+	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+	for (u = 0; u < fm->nhd; u++) {
+		if (fm->hd[u].b == NULL)
+			continue;
+		if (fm->hdf[u] & HDF_FILTER)
+			continue;
+#define HTTPH(a, b, c, d, e, f, g) \
+		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
+			continue;
+#include "http_headers.h"
+#undef HTTPH
+		l += Tlen(fm->hd[u]) + 1;
+		// fm->hdf[u] |= HDF_COPY;
+	}
+	return (l);
+}
+
 /*--------------------------------------------------------------------*/
 
 void

bin/varnishd/flint.lnt

@@ -131,7 +131,9 @@
 // cache_center.c
 -efunc(525, CNT_Session)	// Negative indentation from line
 -efunc(525, http_FilterFields)	// Negative indentation from line
+-efunc(525, http_EstimateWS)	// Negative indentation from line
 -efunc(539, http_FilterFields)	// Positive indentation from line
+-efunc(539, http_EstimateWS)	// Positive indentation from line
 
 // cache_vcl.c
 -efunc(525, vcl_handlingname)	// Negative indentation from line

bin/varnishd/hash_slinger.h

@@ -50,7 +50,7 @@ struct hash_slinger {
 };
 
 /* cache_hash.c */
-struct object *HSH_NewObject(struct sess *sp);
+struct object *HSH_NewObject(struct sess *sp, unsigned len);
 void HSH_Object(const struct sess *sp);
 void HSH_Prealloc(const struct sess *sp);
 void HSH_Cleanup(struct worker *w);

bin/varnishd/mgt_param.c

@@ -492,14 +492,14 @@ static const struct parspec input_parspec[] = {
 		"Minimum is 1024 bytes.",
 		DELAYED_EFFECT,
 		"16384", "bytes" },
-	{ "obj_workspace", tweak_uint, &master.obj_workspace, 1024, UINT_MAX,
+	{ "obj_workspace", tweak_uint, &master.obj_workspace, 0, UINT_MAX,
 		"Bytes of HTTP protocol workspace allocated for objects. "
 		"This space must be big enough for the entire HTTP protocol "
 		"header and any edits done to it in the VCL code while it "
 		"is cached.\n"
 		"Minimum is 1024 bytes.",
 		DELAYED_EFFECT,
-		"8192", "bytes" },
+		"0", "bytes" },
 	{ "shm_workspace", tweak_uint, &master.shm_workspace, 4096, UINT_MAX,
 		"Bytes of shmlog workspace allocated for worker threads. "
 		"If too big, it wastes some ram, if too small it causes "