From apj at varnish-cache.org  Sun Mar  4 15:48:53 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Sun, 04 Mar 2012 16:48:53 +0100
Subject: [master] 86e85bb 3.0 syntax
Message-ID: <E1S4Dg5-0003ZW-6u@project.varnish-software.com>

commit 86e85bb6d95785709566cdfd5aab6db24945f3a9
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sun Mar 4 16:48:45 2012 +0100

    3.0 syntax

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index 83236d6..73d983e 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -146,7 +146,7 @@ header. You could easily add support for this header in VCL.
 In vcl_fetch::
 
   if (beresp.http.Pragma ~ "nocache") {
-     pass;
+     return(pass);
   }
 
 Authorization



From apj at varnish-cache.org  Sun Mar  4 15:52:01 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Sun, 04 Mar 2012 16:52:01 +0100
Subject: [master] 5c9d66c Really change to 3.0 syntax
Message-ID: <E1S4Dj7-00052r-JR@project.varnish-software.com>

commit 5c9d66ce9fdb31a73d1371194ea97a0289db60b2
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sun Mar 4 16:51:38 2012 +0100

    Really change to 3.0 syntax

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index 73d983e..b9fa7e6 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -146,7 +146,7 @@ header. You could easily add support for this header in VCL.
 In vcl_fetch::
 
   if (beresp.http.Pragma ~ "nocache") {
-     return(pass);
+     return(hit_for_pass);
   }
 
 Authorization



From phk at varnish-cache.org  Mon Mar  5 10:37:14 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 05 Mar 2012 11:37:14 +0100
Subject: [master] 5cd69e0 Eliminate obj arg to FetchBody()
Message-ID: <E1S4VI2-0005C1-0n@project.varnish-software.com>

commit 5cd69e085bf15d84ad30233a72c52817def66ca2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 5 10:36:52 2012 +0000

    Eliminate obj arg to FetchBody()
    
    Mostly by:	martin

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bdc3659..dceac39 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -748,7 +748,7 @@ struct storage *FetchStorage(struct busyobj *, ssize_t sz);
 int FetchError(struct busyobj *, const char *error);
 int FetchError2(struct busyobj *, const char *error, const char *more);
 int FetchHdr(struct sess *sp, int need_host_hdr, int sendbody);
-int FetchBody(struct worker *w, struct busyobj *bo, struct object *obj);
+int FetchBody(struct worker *w, struct busyobj *bo);
 int FetchReqBody(const struct sess *sp, int sendbody);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 6e6d54f..9cbc905 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -830,8 +830,10 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req)
 
 	req->storage_hint = NULL;
 
-	if (bo->do_gzip ||
-	    (bo->is_gzip && !bo->do_gunzip))
+	AZ(bo->fetch_obj);
+	bo->fetch_obj = req->obj;
+
+	if (bo->do_gzip || (bo->is_gzip && !bo->do_gunzip))
 		req->obj->gziped = 1;
 
 	if (vary != NULL) {
@@ -903,7 +905,8 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	/* Use unmodified headers*/
-	i = FetchBody(wrk, bo, req->obj);
+	i = FetchBody(wrk, bo);
+	bo->fetch_obj = NULL;
 
 	http_Teardown(bo->bereq);
 	http_Teardown(bo->beresp);
@@ -1115,6 +1118,8 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 
 	/* If we inserted a new object it's a miss */
 	if (oc->flags & OC_F_BUSY) {
+		CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
+		assert(oc->busyobj == req->busyobj);
 		wrk->stats.cache_miss++;
 
 		if (req->vary_l != NULL) {
@@ -1130,7 +1135,6 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 		req->vary_e = NULL;
 
 		req->objcore = oc;
-		CHECK_OBJ_NOTNULL(req->busyobj, BUSYOBJ_MAGIC);
 		sp->step = STP_MISS;
 		return (0);
 	}
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index bb725a2..1dae8b9 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -493,18 +493,19 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody)
 /*--------------------------------------------------------------------*/
 
 int
-FetchBody(struct worker *wrk, struct busyobj *bo, struct object *obj)
+FetchBody(struct worker *wrk, struct busyobj *bo)
 {
 	int cls;
 	struct storage *st;
 	int mklen;
 	ssize_t cl;
 	struct http_conn *htc;
+	struct object *obj;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	AZ(bo->fetch_obj);
 	CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC);
+	obj = bo->fetch_obj;
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
@@ -525,7 +526,6 @@ FetchBody(struct worker *wrk, struct busyobj *bo, struct object *obj)
 	AZ(bo->vgz_rx);
 	AZ(VTAILQ_FIRST(&obj->store));
 
-	bo->fetch_obj = obj;
 	bo->fetch_failed = 0;
 
 	/* XXX: pick up estimate from objdr ? */
@@ -578,8 +578,6 @@ FetchBody(struct worker *wrk, struct busyobj *bo, struct object *obj)
 	 */
 	AZ(vfp_nop_end(bo));
 
-	bo->fetch_obj = NULL;
-
 	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s) cls %d mklen %d",
 	    bo->body_status, body_status(bo->body_status),
 	    cls, mklen);



From phk at varnish-cache.org  Mon Mar  5 11:08:51 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 05 Mar 2012 12:08:51 +0100
Subject: [master] b399a62 Add a set of VFP method functions which can do
	various sanity asserts for us in a single place.
Message-ID: <E1S4Vmd-0007gN-I4@project.varnish-software.com>

commit b399a62ff31a2ee586380b7638193a1e0d0bc236
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 5 11:08:27 2012 +0000

    Add a set of VFP method functions which can do various sanity asserts
    for us in a single place.

diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 14ca31b..ffb90f8 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -346,7 +346,6 @@ vfp_esi_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 	vef = bo->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
-	AZ(bo->fetch_failed);
 	AN(bo->vep);
 	assert(&bo->htc == htc);
 	if (bo->is_gzip && bo->do_gunzip)
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1dae8b9..1313bc1 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -73,6 +73,40 @@ FetchError(struct busyobj *bo, const char *error)
 }
 
 /*--------------------------------------------------------------------
+ * VFP method functions
+ */
+
+static void
+VFP_Begin(struct busyobj *bo, size_t estimate)
+{
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	AN(bo->vfp);
+
+	bo->vfp->begin(bo, estimate);
+}
+
+static int
+VFP_Bytes(struct busyobj *bo, struct http_conn *htc, ssize_t sz)
+{
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	AN(bo->vfp);
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	AZ(bo->fetch_failed);
+
+	return (bo->vfp->bytes(bo, htc, sz));
+}
+
+static int
+VFP_End(struct busyobj *bo)
+{
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	AN(bo->vfp);
+
+	return (bo->vfp->end(bo));
+}
+
+
+/*--------------------------------------------------------------------
  * VFP_NOP
  *
  * This fetch-processor does nothing but store the object.
@@ -235,7 +269,7 @@ fetch_straight(struct busyobj *bo, struct http_conn *htc, ssize_t cl)
 	} else if (cl == 0)
 		return (0);
 
-	i = bo->vfp->bytes(bo, htc, cl);
+	i = VFP_Bytes(bo, htc, cl);
 	if (i <= 0)
 		return (FetchError(bo, "straight insufficient bytes"));
 	return (0);
@@ -292,7 +326,7 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 		if (cl < 0)
 			return (FetchError(bo,"chunked header number syntax"));
 
-		if (cl > 0 && bo->vfp->bytes(bo, htc, cl) <= 0)
+		if (cl > 0 && VFP_Bytes(bo, htc, cl) <= 0)
 			return (-1);
 
 		i = HTC_Read(htc, buf, 1);
@@ -314,7 +348,7 @@ fetch_eof(struct busyobj *bo, struct http_conn *htc)
 	int i;
 
 	assert(bo->body_status == BS_EOF);
-	i = bo->vfp->bytes(bo, htc, SSIZE_MAX);
+	i = VFP_Bytes(bo, htc, SSIZE_MAX);
 	if (i < 0)
 		return (-1);
 	return (0);
@@ -541,24 +575,24 @@ FetchBody(struct worker *wrk, struct busyobj *bo)
 		break;
 	case BS_LENGTH:
 		cl = fetch_number(bo->h_content_length, 10);
-		bo->vfp->begin(bo, cl > 0 ? cl : 0);
+		VFP_Begin(bo, cl > 0 ? cl : 0);
 		cls = fetch_straight(bo, htc, cl);
 		mklen = 1;
-		if (bo->vfp->end(bo))
+		if (VFP_End(bo))
 			cls = -1;
 		break;
 	case BS_CHUNKED:
-		bo->vfp->begin(bo, cl);
+		VFP_Begin(bo, cl);
 		cls = fetch_chunked(bo, htc);
 		mklen = 1;
-		if (bo->vfp->end(bo))
+		if (VFP_End(bo))
 			cls = -1;
 		break;
 	case BS_EOF:
-		bo->vfp->begin(bo, cl);
+		VFP_Begin(bo, cl);
 		cls = fetch_eof(bo, htc);
 		mklen = 1;
-		if (bo->vfp->end(bo))
+		if (VFP_End(bo))
 			cls = -1;
 		break;
 	case BS_ERROR:
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index ec1b7cd..e2b4d51 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -448,7 +448,6 @@ vfp_gunzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 	const void *dp;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	AZ(bo->fetch_failed);
 	vg = bo->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -526,7 +525,6 @@ vfp_gzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 	const void *dp;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	AZ(bo->fetch_failed);
 	vg = bo->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -613,7 +611,6 @@ vfp_testgzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 	struct storage *st;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	AZ(bo->fetch_failed);
 	vg = bo->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);



From phk at varnish-cache.org  Mon Mar  5 11:27:59 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 05 Mar 2012 12:27:59 +0100
Subject: [master] f9e413c Don't assert if we fail to get storage in VFP_Begin()
Message-ID: <E1S4W59-0003Qw-9a@project.varnish-software.com>

commit f9e413c77249b0b6c407c087836a1cd80d72d1ab
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 5 11:27:37 2012 +0000

    Don't assert if we fail to get storage in VFP_Begin()
    
    Fixes	#1100

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1313bc1..1f146d0 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -76,13 +76,16 @@ FetchError(struct busyobj *bo, const char *error)
  * VFP method functions
  */
 
-static void
+static int
 VFP_Begin(struct busyobj *bo, size_t estimate)
 {
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AN(bo->vfp);
 
 	bo->vfp->begin(bo, estimate);
+	if (bo->fetch_failed)
+		return (-1);
+	return (0);
 }
 
 static int
@@ -146,7 +149,6 @@ vfp_nop_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 	ssize_t l, wl;
 	struct storage *st;
 
-	AZ(bo->fetch_failed);
 	while (bytes > 0) {
 		st = FetchStorage(bo, 0);
 		if (st == NULL)
@@ -575,22 +577,25 @@ FetchBody(struct worker *wrk, struct busyobj *bo)
 		break;
 	case BS_LENGTH:
 		cl = fetch_number(bo->h_content_length, 10);
-		VFP_Begin(bo, cl > 0 ? cl : 0);
-		cls = fetch_straight(bo, htc, cl);
+		cls = VFP_Begin(bo, cl > 0 ? cl : 0);
+		if (!cls)
+			cls = fetch_straight(bo, htc, cl);
 		mklen = 1;
 		if (VFP_End(bo))
 			cls = -1;
 		break;
 	case BS_CHUNKED:
-		VFP_Begin(bo, cl);
-		cls = fetch_chunked(bo, htc);
+		cls = VFP_Begin(bo, cl);
+		if (!cls)
+			cls = fetch_chunked(bo, htc);
 		mklen = 1;
 		if (VFP_End(bo))
 			cls = -1;
 		break;
 	case BS_EOF:
-		VFP_Begin(bo, cl);
-		cls = fetch_eof(bo, htc);
+		cls = VFP_Begin(bo, cl);
+		if (!cls)
+			cls = fetch_eof(bo, htc);
 		mklen = 1;
 		if (VFP_End(bo))
 			cls = -1;
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 2689cc8..c5f49de 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -213,7 +213,8 @@ pan_busyobj(const struct busyobj *bo)
 	if (bo->do_esi)		VSB_printf(pan_vsp, "    do_esi\n");
 	if (bo->do_stream)	VSB_printf(pan_vsp, "    do_stream\n");
 	if (bo->should_close)	VSB_printf(pan_vsp, "    should_close\n");
-	VSB_printf(pan_vsp, "    bodystatus = %d,\n", bo->body_status);
+	VSB_printf(pan_vsp, "    bodystatus = %d (%s),\n",
+	    bo->body_status, body_status(bo->body_status));
 	VSB_printf(pan_vsp, "    },\n");
 	if (VALID_OBJ(bo->vbc, BACKEND_MAGIC))
 		pan_vbc(bo->vbc);



From phk at varnish-cache.org  Mon Mar  5 12:12:59 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 05 Mar 2012 13:12:59 +0100
Subject: [master] 501af0b Add a bo->state that tells us what's going on.
	Improve error reporting at the same time.
Message-ID: <E1S4Wmh-0006UK-SH@project.varnish-software.com>

commit 501af0ba839629de342a2c17963bb04de8599625
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 5 12:12:37 2012 +0000

    Add a bo->state that tells us what's going on.
    Improve error reporting at the same time.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index dceac39..bea32a2 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -449,6 +449,13 @@ oc_getlru(const struct objcore *oc)
  * streaming delivery will make use of.
  */
 
+enum busyobj_state_e {
+	BOS_INVALID = 0,
+	BOS_FETCHING,
+	BOS_FAILED,
+	BOS_FINISHED
+};
+
 struct busyobj {
 	unsigned		magic;
 #define BUSYOBJ_MAGIC		0x23b95567
@@ -460,7 +467,7 @@ struct busyobj {
 
 	struct vfp		*vfp;
 	struct vep_state	*vep;
-	unsigned		fetch_failed;
+	enum busyobj_state_e	state;
 	struct vgz		*vgz_rx;
 
 	struct ws		ws[1];
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index ffb90f8..6699117 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -366,12 +366,13 @@ vfp_esi_end(struct busyobj *bo)
 	struct vsb *vsb;
 	struct vef_priv *vef;
 	ssize_t l;
-	int retval;
+	int retval = 0;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AN(bo->vep);
 
-	retval = bo->fetch_failed;
+	if (bo->state == BOS_FAILED)
+		retval = -1;
 
 	if (bo->vgz_rx != NULL && VGZ_Destroy(&bo->vgz_rx) != VGZ_END)
 		retval = FetchError(bo, "Gunzip+ESI Failed at the very end");
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1f146d0..fd70d07 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -56,13 +56,13 @@ FetchError2(struct busyobj *bo, const char *error, const char *more)
 {
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	if (!bo->fetch_failed) {
+	if (bo->state == BOS_FETCHING) {
 		if (more == NULL)
 			VSLb(bo->vsl, SLT_FetchError, "%s", error);
 		else
 			VSLb(bo->vsl, SLT_FetchError, "%s: %s", error, more);
 	}
-	bo->fetch_failed = 1;
+	bo->state = BOS_FAILED;
 	return (-1);
 }
 
@@ -83,7 +83,7 @@ VFP_Begin(struct busyobj *bo, size_t estimate)
 	AN(bo->vfp);
 
 	bo->vfp->begin(bo, estimate);
-	if (bo->fetch_failed)
+	if (bo->state == BOS_FAILED)
 		return (-1);
 	return (0);
 }
@@ -94,7 +94,7 @@ VFP_Bytes(struct busyobj *bo, struct http_conn *htc, ssize_t sz)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AN(bo->vfp);
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	AZ(bo->fetch_failed);
+	assert(bo->state == BOS_FETCHING);
 
 	return (bo->vfp->bytes(bo, htc, sz));
 }
@@ -102,10 +102,16 @@ VFP_Bytes(struct busyobj *bo, struct http_conn *htc, ssize_t sz)
 static int
 VFP_End(struct busyobj *bo)
 {
+	int i;
+
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AN(bo->vfp);
 
-	return (bo->vfp->end(bo));
+	i = bo->vfp->end(bo);
+	if (i)
+		assert(bo->state == BOS_FAILED);
+	return (i);
+
 }
 
 
@@ -295,8 +301,8 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 	do {
 		/* Skip leading whitespace */
 		do {
-			if (HTC_Read(htc, buf, 1) <= 0)
-				return (-1);
+			if (HTC_Read(htc, buf, 1) <= 0) 
+				return (FetchError(bo, "chunked read err"));
 		} while (vct_islws(buf[0]));
 
 		if (!vct_ishex(buf[0]))
@@ -306,7 +312,8 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 		for (u = 1; u < sizeof buf; u++) {
 			do {
 				if (HTC_Read(htc, buf + u, 1) <= 0)
-					return (-1);
+					return (FetchError(bo,
+					    "chunked read err"));
 			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
 			if (!vct_ishex(buf[u]))
 				break;
@@ -318,7 +325,7 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 		/* Skip trailing white space */
 		while(vct_islws(buf[u]) && buf[u] != '\n')
 			if (HTC_Read(htc, buf + u, 1) <= 0)
-				return (-1);
+				return (FetchError(bo, "chunked read err"));
 
 		if (buf[u] != '\n')
 			return (FetchError(bo,"chunked header no NL"));
@@ -329,13 +336,13 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 			return (FetchError(bo,"chunked header number syntax"));
 
 		if (cl > 0 && VFP_Bytes(bo, htc, cl) <= 0)
-			return (-1);
+			return (FetchError(bo, "chunked read err"));
 
 		i = HTC_Read(htc, buf, 1);
 		if (i <= 0)
-			return (-1);
+			return (FetchError(bo, "chunked read err"));
 		if (buf[0] == '\r' && HTC_Read( htc, buf, 1) <= 0)
-			return (-1);
+			return (FetchError(bo, "chunked read err"));
 		if (buf[0] != '\n')
 			return (FetchError(bo,"chunked tail no NL"));
 	} while (cl > 0);
@@ -352,7 +359,7 @@ fetch_eof(struct busyobj *bo, struct http_conn *htc)
 	assert(bo->body_status == BS_EOF);
 	i = VFP_Bytes(bo, htc, SSIZE_MAX);
 	if (i < 0)
-		return (-1);
+		return (FetchError(bo,"eof socket fail"));
 	return (0);
 }
 
@@ -545,6 +552,9 @@ FetchBody(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
+	assert(bo->state == BOS_INVALID);
+	bo->state = BOS_FINISHED;
+
 	/*
 	 * XXX: The busyobj needs a dstat, but it is not obvious which one
 	 * XXX: it should be (own/borrowed).  For now borrow the wrk's.
@@ -562,7 +572,7 @@ FetchBody(struct worker *wrk, struct busyobj *bo)
 	AZ(bo->vgz_rx);
 	AZ(VTAILQ_FIRST(&obj->store));
 
-	bo->fetch_failed = 0;
+	bo->state = BOS_FETCHING;
 
 	/* XXX: pick up estimate from objdr ? */
 	cl = 0;
@@ -628,13 +638,14 @@ FetchBody(struct worker *wrk, struct busyobj *bo)
 	}
 
 	if (cls < 0) {
+		assert(bo->state == BOS_FAILED);
 		wrk->stats.fetch_failed++;
 		VDI_CloseFd(&bo->vbc);
 		obj->len = 0;
 		bo->stats = NULL;
 		return (__LINE__);
 	}
-	AZ(bo->fetch_failed);
+	assert(bo->state == BOS_FETCHING);
 
 	if (cls == 0 && bo->should_close)
 		cls = 1;
@@ -661,6 +672,8 @@ FetchBody(struct worker *wrk, struct busyobj *bo)
 		http_PrintfHeader(obj->http, "Content-Length: %zd", obj->len);
 	}
 
+	bo->state = BOS_FINISHED;
+
 	if (cls)
 		VDI_CloseFd(&bo->vbc);
 	else
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index e2b4d51..343bd35 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -483,7 +483,7 @@ vfp_gunzip_end(struct busyobj *bo)
 	vg = bo->vgz_rx;
 	bo->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	if (bo->fetch_failed) {
+	if (bo->state == BOS_FAILED) {
 		(void)VGZ_Destroy(&vg);
 		return(0);
 	}
@@ -560,7 +560,7 @@ vfp_gzip_end(struct busyobj *bo)
 	vg = bo->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	bo->vgz_rx = NULL;
-	if (bo->fetch_failed) {
+	if (bo->state == BOS_FAILED) {
 		(void)VGZ_Destroy(&vg);
 		return(0);
 	}
@@ -652,7 +652,7 @@ vfp_testgzip_end(struct busyobj *bo)
 	vg = bo->vgz_rx;
 	bo->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	if (bo->fetch_failed) {
+	if (bo->state == BOS_FAILED) {
 		(void)VGZ_Destroy(&vg);
 		return(0);
 	}



From phk at varnish-cache.org  Mon Mar  5 14:19:32 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 05 Mar 2012 15:19:32 +0100
Subject: [master] 071d905 Eliminate the returnvalue from FetchBody(),
	rely on bo->state instead
Message-ID: <E1S4YlA-0005YX-C1@project.varnish-software.com>

commit 071d9050d5285bc8caa511c7664f00051f844496
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 5 14:19:10 2012 +0000

    Eliminate the returnvalue from FetchBody(), rely on bo->state instead

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bea32a2..63ea682 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -755,7 +755,7 @@ struct storage *FetchStorage(struct busyobj *, ssize_t sz);
 int FetchError(struct busyobj *, const char *error);
 int FetchError2(struct busyobj *, const char *error, const char *more);
 int FetchHdr(struct sess *sp, int need_host_hdr, int sendbody);
-int FetchBody(struct worker *w, struct busyobj *bo);
+void FetchBody(struct worker *w, struct busyobj *bo);
 int FetchReqBody(const struct sess *sp, int sendbody);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 9cbc905..d13fc46 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -895,7 +895,6 @@ DOT fetchbody:out -> prepresp [style=bold,color=blue]
 static int
 cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	int i;
 	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -904,8 +903,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	/* Use unmodified headers*/
-	i = FetchBody(wrk, bo);
+	FetchBody(wrk, bo);
 	bo->fetch_obj = NULL;
 
 	http_Teardown(bo->bereq);
@@ -915,7 +913,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	AZ(bo->vbc);
 	AN(req->director);
 
-	if (i) {
+	if (bo->state == BOS_FAILED) {
 		HSH_Drop(wrk, &sp->req->obj);
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		AZ(req->obj);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index fd70d07..4fc0f8e 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -76,21 +76,20 @@ FetchError(struct busyobj *bo, const char *error)
  * VFP method functions
  */
 
-static int
+static void
 VFP_Begin(struct busyobj *bo, size_t estimate)
 {
+
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AN(bo->vfp);
 
 	bo->vfp->begin(bo, estimate);
-	if (bo->state == BOS_FAILED)
-		return (-1);
-	return (0);
 }
 
 static int
 VFP_Bytes(struct busyobj *bo, struct http_conn *htc, ssize_t sz)
 {
+
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AN(bo->vfp);
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
@@ -99,7 +98,7 @@ VFP_Bytes(struct busyobj *bo, struct http_conn *htc, ssize_t sz)
 	return (bo->vfp->bytes(bo, htc, sz));
 }
 
-static int
+static void
 VFP_End(struct busyobj *bo)
 {
 	int i;
@@ -110,8 +109,6 @@ VFP_End(struct busyobj *bo)
 	i = bo->vfp->end(bo);
 	if (i)
 		assert(bo->state == BOS_FAILED);
-	return (i);
-
 }
 
 
@@ -351,16 +348,13 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 
 /*--------------------------------------------------------------------*/
 
-static int
+static void
 fetch_eof(struct busyobj *bo, struct http_conn *htc)
 {
-	int i;
 
 	assert(bo->body_status == BS_EOF);
-	i = VFP_Bytes(bo, htc, SSIZE_MAX);
-	if (i < 0)
-		return (FetchError(bo,"eof socket fail"));
-	return (0);
+	if (VFP_Bytes(bo, htc, SSIZE_MAX) < 0)
+		(void)FetchError(bo,"eof socket fail");
 }
 
 /*--------------------------------------------------------------------
@@ -535,7 +529,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody)
 
 /*--------------------------------------------------------------------*/
 
-int
+void
 FetchBody(struct worker *wrk, struct busyobj *bo)
 {
 	int cls;
@@ -576,42 +570,39 @@ FetchBody(struct worker *wrk, struct busyobj *bo)
 
 	/* XXX: pick up estimate from objdr ? */
 	cl = 0;
+	cls = 0;
 	switch (bo->body_status) {
 	case BS_NONE:
-		cls = 0;
 		mklen = 0;
 		break;
 	case BS_ZERO:
-		cls = 0;
 		mklen = 1;
 		break;
 	case BS_LENGTH:
 		cl = fetch_number(bo->h_content_length, 10);
-		cls = VFP_Begin(bo, cl > 0 ? cl : 0);
-		if (!cls)
+		VFP_Begin(bo, cl > 0 ? cl : 0);
+		if (bo->state == BOS_FETCHING)
 			cls = fetch_straight(bo, htc, cl);
 		mklen = 1;
-		if (VFP_End(bo))
-			cls = -1;
+		VFP_End(bo);
 		break;
 	case BS_CHUNKED:
-		cls = VFP_Begin(bo, cl);
-		if (!cls)
+		VFP_Begin(bo, cl);
+		if (bo->state == BOS_FETCHING)
 			cls = fetch_chunked(bo, htc);
 		mklen = 1;
-		if (VFP_End(bo))
-			cls = -1;
+		VFP_End(bo);
 		break;
 	case BS_EOF:
-		cls = VFP_Begin(bo, cl);
-		if (!cls)
-			cls = fetch_eof(bo, htc);
+		VFP_Begin(bo, cl);
+		if (bo->state == BOS_FETCHING)
+			fetch_eof(bo, htc);
 		mklen = 1;
-		if (VFP_End(bo))
-			cls = -1;
+		cls = 1;
+		VFP_End(bo);
 		break;
 	case BS_ERROR:
-		cls = 1;
+		cls = FetchError(bo, "error incompatible Transfer-Encoding");
 		mklen = 0;
 		break;
 	default:
@@ -631,20 +622,14 @@ FetchBody(struct worker *wrk, struct busyobj *bo)
 	    bo->body_status, body_status(bo->body_status),
 	    cls, mklen);
 
-	if (bo->body_status == BS_ERROR) {
-		VDI_CloseFd(&bo->vbc);
-		bo->stats = NULL;
-		return (__LINE__);
-	}
-
-	if (cls < 0) {
-		assert(bo->state == BOS_FAILED);
+	if (bo->state == BOS_FAILED) {
 		wrk->stats.fetch_failed++;
 		VDI_CloseFd(&bo->vbc);
 		obj->len = 0;
 		bo->stats = NULL;
-		return (__LINE__);
+		return;
 	}
+
 	assert(bo->state == BOS_FETCHING);
 
 	if (cls == 0 && bo->should_close)
@@ -680,7 +665,6 @@ FetchBody(struct worker *wrk, struct busyobj *bo)
 		VDI_RecycleFd(&bo->vbc);
 
 	bo->stats = NULL;
-	return (0);
 }
 
 /*--------------------------------------------------------------------



From phk at varnish-cache.org  Mon Mar  5 14:38:38 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 05 Mar 2012 15:38:38 +0100
Subject: [master] 8f2d2e7 Make FetchBody() pool-task compatible.
Message-ID: <E1S4Z3e-0005ar-MC@project.varnish-software.com>

commit 8f2d2e7bce9bf5a614a42a296b0b49d67ec4be25
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 5 14:37:54 2012 +0000

    Make FetchBody() pool-task compatible.
    
    Add #ifdef 0'ed code to put another thread on body-fetching
    
    Hugely inpired by patches from:	martin

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 63ea682..2bae6b4 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -479,6 +479,8 @@ struct busyobj {
 	struct http_conn	htc;
 
 	enum body_status	body_status;
+	struct pool_task	task;
+
 	struct vef_priv		*vef_priv;
 
 	unsigned		should_close;
@@ -755,7 +757,7 @@ struct storage *FetchStorage(struct busyobj *, ssize_t sz);
 int FetchError(struct busyobj *, const char *error);
 int FetchError2(struct busyobj *, const char *error, const char *more);
 int FetchHdr(struct sess *sp, int need_host_hdr, int sendbody);
-void FetchBody(struct worker *w, struct busyobj *bo);
+void FetchBody(struct worker *w, void *bo);
 int FetchReqBody(const struct sess *sp, int sendbody);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index d13fc46..ecea4b5 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -903,8 +903,19 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+#if 1
 	FetchBody(wrk, bo);
-	bo->fetch_obj = NULL;
+#else
+	bo->task.func = FetchBody;
+	bo->task.priv = bo;
+	if (Pool_Task(wrk->pool, &bo->task, POOL_NO_QUEUE)) {
+		FetchBody(wrk, bo);
+	} else {
+		while (bo->state < BOS_FAILED)
+			(void)usleep(10000);
+	}
+#endif
+	assert(bo->state >= BOS_FAILED);
 
 	http_Teardown(bo->bereq);
 	http_Teardown(bo->beresp);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 4fc0f8e..88bca02 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -530,7 +530,7 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody)
 /*--------------------------------------------------------------------*/
 
 void
-FetchBody(struct worker *wrk, struct busyobj *bo)
+FetchBody(struct worker *wrk, void *priv)
 {
 	int cls;
 	struct storage *st;
@@ -538,9 +538,10 @@ FetchBody(struct worker *wrk, struct busyobj *bo)
 	ssize_t cl;
 	struct http_conn *htc;
 	struct object *obj;
+	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC);
 	obj = bo->fetch_obj;
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);



From tfheen at varnish-cache.org  Wed Mar  7 13:18:34 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 07 Mar 2012 14:18:34 +0100
Subject: [master] af587ef Grammar in varnishncsa man page
Message-ID: <E1S5GlG-0002Of-Ux@project.varnish-software.com>

commit af587effb70d2dfbffd45bb54abe7dca2bf08293
Author: Tollef Fog Heen <tfheen at err.no>
Date:   Wed Mar 7 14:18:24 2012 +0100

    Grammar in varnishncsa man page

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 9373f5c..2937b26 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -64,7 +64,7 @@ The following options are available:
                  Defaults to 127.0.0.1 for backend requests.
 
 	      %{X}i
-	         The contents of request header line X.
+	         The contents of request header X.
 
 	      %l
 	         Remote logname (always '-')
@@ -77,7 +77,7 @@ The following options are available:
                  empty string.
 
 	      %{X}o
-	         The contents of response header line X.
+	         The contents of response header X.
 
 	      %r
 	         The first line of the request. Synthesized from other



From tfheen at varnish-cache.org  Thu Mar  8 10:37:43 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 08 Mar 2012 11:37:43 +0100
Subject: [master] fe828b4 Fixing a typo in the documentation.
Message-ID: <E1S5aj9-0007Y4-7w@project.varnish-software.com>

commit fe828b4cfd698c99be3b4e09dc2acc21a88caaa3
Author: Derek Hammer <derek.r.hammer at gmail.com>
Date:   Thu Mar 8 03:11:26 2012 -0500

    Fixing a typo in the documentation.

diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst
index ecf0e96..846ca82 100644
--- a/doc/sphinx/tutorial/purging.rst
+++ b/doc/sphinx/tutorial/purging.rst
@@ -4,7 +4,7 @@
  Purging and banning
 =====================
 
-One of the most effective way of increasing your hit ratio is to
+One of the most effective ways of increasing your hit ratio is to
 increase the time-to-live (ttl) of your objects. But, as you're aware
 of, in this twitterific day of age serving content that is outdated is
 bad for business.



From lkarsten at varnish-cache.org  Thu Mar  8 13:30:54 2012
From: lkarsten at varnish-cache.org (Lasse Karstensen)
Date: Thu, 08 Mar 2012 14:30:54 +0100
Subject: [master] 4fe672c Add section on device detection
Message-ID: <E1S5dQk-0004rQ-Rs@project.varnish-software.com>

commit 4fe672ca2b418043a965ab28049e1256a5011dbe
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Mar 7 14:35:41 2012 +0100

    Add section on device detection

diff --git a/doc/sphinx/tutorial/devicedetection.rst b/doc/sphinx/tutorial/devicedetection.rst
new file mode 100644
index 0000000..a083e6e
--- /dev/null
+++ b/doc/sphinx/tutorial/devicedetection.rst
@@ -0,0 +1,260 @@
+.. _tutorial-devicedetect:
+
+Device detection
+~~~~~~~~~~~~~~~~
+
+Device detection is figuring out what kind of content to serve to a
+client based on the User-Agent string supplied in a request.
+
+Use cases for this are for example to send size reduced files to mobile
+clients with small screens and on high latency networks, or to 
+provide a streaming video codec that the client understands.
+
+There are a couple of strategies on what to do with such clients:
+1) Redirect them to another URL.
+2) Use a different backend for the special clients.
+3) Change the backend requests so the usual backend sends tailored content.
+
+To make the examples easier to understand, it is assumed in this text 
+that all the req.http.X-UA-Device header is present and unique per client class
+that content is to be served to. 
+
+Setting this header can be as simple as::
+
+   sub vcl_recv {?
+       if (req.http.User-Agent ~ "(?i)iphone"?{
+           set req.http.X-UA-Device = "mobile-iphone";
+       }
+   }
+
+There are different commercial and free offerings in doing grouping and identifiying clients
+in further detail than this.
+
+
+Serve the different content on the same URL
+-------------------------------------------
+
+The tricks involved are: 
+1. Detect the client (pretty simple, just include devicedetect.vcl and call
+it)
+2. Figure out how to signal the backend what client class this is. This
+includes for example setting a header, changing a header or even changing the
+backend request URL.
+3. Modify any response from the backend to add missing Vary headers, so
+Varnish' internal handling of this kicks in.
+4. Modify output sent to the client so any caches outside our control don't
+serve the wrong content.
+
+All this while still making sure that we only get 1 cache object per URL per
+device class.
+
+
+Example 1: Send HTTP header to backend
+''''''''''''''''''''''''''''''''''''''
+
+The basic case is that Varnish add the X-UA-Device HTTP header on the backend
+requests, and the backend mentions in the response Vary header that the content
+is dependant on this header. 
+
+Everything works out of the box from Varnish' perspective.
+
+.. 071-example1-start
+VCL::
+
+    sub vcl_recv { 
+        # call some detection engine that set req.http.X-UA-Device
+    }
+
+    sub append_ua_device {
+        if (req.http.X-UA-Device) { 
+            set bereq.http.X-UA-Device = req.http.X-UA-Device; }
+    }
+
+    # This must be done in vcl_miss and vcl_pass, before any backend request is
+    # actually sent. vcl_fetch runs after the request to the backend has
+    # completed.
+    sub vcl_miss { call append_ua_device; }
+    sub vcl_pass { call append_ua_device; }
+
+    # so, this is a bit conterintuitive. The backend creates content based on
+    # the normalized User-Agent, but we use Vary on X-UA-Device so Varnish will
+    # use the same cached object for all U-As that map to the same X-UA-Device.
+    #
+    # If the backend does not mention in Vary that it has crafted special
+    # content based on the User-Agent (==X-UA-Device), add it. 
+    # If your backend does set Vary: User-Agent, you may have to remove that here.
+    sub vcl_fetch {
+        if (req.http.X-UA-Device) {
+            if (!beresp.http.Vary) { # no Vary at all
+                set beresp.http.Vary = "X-UA-Device"; 
+            } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
+                set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device"; 
+            } 
+        }
+        # comment this out if you don't want the client to know your
+        # classification
+        set beresp.http.X-UA-Device = req.http.X-UA-Device;
+    }
+
+    # to keep any caches in the wild from serving wrong content to client #2
+    # behind them, we need to transform the Vary on the way out.
+    sub vcl_deliver {
+        if ((req.http.X-UA-Device) && (resp.http.Vary)) {
+            set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent");
+        }
+    }
+.. 071-example1-end
+
+Example 2: Normalize the User-Agent string
+''''''''''''''''''''''''''''''''''''''''''
+
+Another way of signaling the device type is to override or normalize the
+User-Agent header sent to the backend.
+
+For example
+
+    User-Agent: Mozilla/5.0 (Linux; U; Android 2.2; nb-no; HTC Desire Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1
+
+becomes:
+
+    User-Agent: mobile-android
+
+when seen by the backend.
+
+This works if you don't need the original header for anything on the backend.
+A possible use for this is for CGI scripts where only a small set of predefined
+headers are (by default) available for the script.
+
+.. 072-example2-start
+VCL::
+
+    sub vcl_recv { 
+        # call some detection engine that set req.http.X-UA-Device
+    }
+
+    # override the header before it is sent to the backend
+    sub vcl_miss { if (req.http.X-UA-Device) { set bereq.http.User-Agent = req.http.X-UA-Device; } }
+    sub vcl_pass { if (req.http.X-UA-Device) { set bereq.http.User-Agent = req.http.X-UA-Device; } }
+
+    # standard Vary handling code from previous examples.
+    sub vcl_fetch {
+        if (req.http.X-UA-Device) {
+            if (!beresp.http.Vary) { # no Vary at all
+                set beresp.http.Vary = "X-UA-Device";
+            } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
+                set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device";
+            }
+        }
+        set beresp.http.X-UA-Device = req.http.X-UA-Device;
+    }
+    sub vcl_deliver {
+        if ((req.http.X-UA-Device) && (resp.http.Vary)) {
+            set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent");
+        }
+    }
+
+.. 072-example2-end
+
+Example 3: Add the device class as a GET query parameter
+''''''''''''''''''''''''''''''''''''''''''''''''''''''''
+
+If everything else fails, you can add the device type as a GET argument.
+
+    http://example.com/article/1234.html --> http://example.com/article/1234.html?devicetype=mobile-iphone
+
+The client itself does not see this classification, only the backend request
+is changed.
+
+.. 073-example3-start
+VCL::
+
+    sub vcl_recv { 
+        # call some detection engine that set req.http.X-UA-Device
+
+        if ((req.http.X-UA-Device) && (req.request == "GET")) {
+            # if there are existing GET arguments;
+            if (req.url ~ "\?") {
+                set req.http.X-get-devicetype = "&devicetype=" + req.http.X-UA-Device;
+            } else { 
+                set req.http.X-get-devicetype = "?devicetype=" + req.http.X-UA-Device;
+            }
+            set req.url = req.url + req.http.X-get-devicetype;
+            unset req.http.X-get-devicetype;
+        }
+    }
+
+    # Handle redirects, otherwise standard Vary handling code from previous
+    # examples.
+    sub vcl_fetch {
+        if (req.http.X-UA-Device) {
+            if (!beresp.http.Vary) { # no Vary at all
+                set beresp.http.Vary = "X-UA-Device";
+            } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
+                set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device";
+            }
+
+            # if the backend returns a redirect (think missing trailing slash),
+            # we will potentially show the extra address to the client. we
+            # don't want that.  if the backend reorders the get parameters, you
+            # may need to be smarter here. (? and & ordering)
+
+            if (beresp.status == 301 || beresp.status == 302 || beresp.status == 303) {
+                set beresp.http.location = regsub(beresp.http.location, "[?&]devicetype=.*$", "");
+            }
+        }
+        set beresp.http.X-UA-Device = req.http.X-UA-Device;
+    }
+    sub vcl_deliver {
+        if ((req.http.X-UA-Device) && (resp.http.Vary)) {
+            set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent");
+        }
+    }
+
+.. 073-example3-end
+
+Different backend for mobile clients
+------------------------------------
+
+If you have a different backend that serves pages for mobile clients, or any
+special needs in VCL, you can use the X-UA-Device header like this::
+
+    backend mobile {
+        .host = "10.0.0.1";
+        .port = "80";
+    }
+
+    sub vcl_recv {
+        # call some detection engine
+
+        if (req.http.X-UA-Device ~ "^mobile" || req.http.X-UA-device ~ "^tablet") {
+            set req.backend = mobile;
+        }
+    }
+
+Redirecting mobile clients
+--------------------------
+
+If you want to redirect mobile clients you can use the following snippet.
+
+.. 065-redir-mobile-start
+VCL::
+
+    sub vcl_recv {
+        # call some detection engine
+
+        if (req.http.X-UA-Device ~ "^mobile" || req.http.X-UA-device ~ "^tablet") {
+            error 750 "Moved Temporarily";
+        }
+    }
+     
+    sub vcl_error {
+        if (obj.status == 750) {
+            set obj.http.Location = "http://m.example.com" + req.url;
+            set obj.status = 302;
+            return(deliver);
+        }
+    }
+
+.. 065-redir-mobile-end
+
+
diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 4949aa0..91fdb17 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -30,6 +30,7 @@ separate topic. Good luck.
 	esi
 	virtualized
 	websockets
+	devicedetection
 	advanced_backend_servers
         handling_misbehaving_servers
         advanced_topics



From lkarsten at varnish-cache.org  Thu Mar  8 13:30:54 2012
From: lkarsten at varnish-cache.org (Lasse Karstensen)
Date: Thu, 08 Mar 2012 14:30:54 +0100
Subject: [master] 72cb9f6 typos, change example3 to allow for easier purging
Message-ID: <E1S5dQk-0004rX-T2@project.varnish-software.com>

commit 72cb9f69167eff08e6498db0ce732eac0e581b20
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Thu Mar 8 14:26:27 2012 +0100

    typos, change example3 to allow for easier purging

diff --git a/doc/sphinx/tutorial/devicedetection.rst b/doc/sphinx/tutorial/devicedetection.rst
index a083e6e..abf729e 100644
--- a/doc/sphinx/tutorial/devicedetection.rst
+++ b/doc/sphinx/tutorial/devicedetection.rst
@@ -27,8 +27,10 @@ Setting this header can be as simple as::
        }
    }
 
-There are different commercial and free offerings in doing grouping and identifiying clients
-in further detail than this.
+There are different commercial and free offerings in doing grouping and
+identifiying clients in further detail than this. For a basic and community
+based regular expression set, see
+https://github.com/varnish/varnish-devicedetect/ .
 
 
 Serve the different content on the same URL
@@ -45,14 +47,14 @@ Varnish' internal handling of this kicks in.
 4. Modify output sent to the client so any caches outside our control don't
 serve the wrong content.
 
-All this while still making sure that we only get 1 cache object per URL per
+All this while still making sure that we only get 1 cached object per URL per
 device class.
 
 
 Example 1: Send HTTP header to backend
 ''''''''''''''''''''''''''''''''''''''
 
-The basic case is that Varnish add the X-UA-Device HTTP header on the backend
+The basic case is that Varnish adds the X-UA-Device HTTP header on the backend
 requests, and the backend mentions in the response Vary header that the content
 is dependant on this header. 
 
@@ -64,17 +66,7 @@ VCL::
     sub vcl_recv { 
         # call some detection engine that set req.http.X-UA-Device
     }
-
-    sub append_ua_device {
-        if (req.http.X-UA-Device) { 
-            set bereq.http.X-UA-Device = req.http.X-UA-Device; }
-    }
-
-    # This must be done in vcl_miss and vcl_pass, before any backend request is
-    # actually sent. vcl_fetch runs after the request to the backend has
-    # completed.
-    sub vcl_miss { call append_ua_device; }
-    sub vcl_pass { call append_ua_device; }
+    # req.http.X-UA-Device is copied by Varnish into bereq.http.X-UA-Device
 
     # so, this is a bit conterintuitive. The backend creates content based on
     # the normalized User-Agent, but we use Vary on X-UA-Device so Varnish will
@@ -170,7 +162,9 @@ VCL::
 
     sub vcl_recv { 
         # call some detection engine that set req.http.X-UA-Device
+    }
 
+    sub append_ua {
         if ((req.http.X-UA-Device) && (req.request == "GET")) {
             # if there are existing GET arguments;
             if (req.url ~ "\?") {
@@ -183,6 +177,10 @@ VCL::
         }
     }
 
+    # do this after vcl_hash, so all Vary-ants can be purged in one go. (avoid ban()ing)
+    sub vcl_miss { call append_ua; }
+    sub vcl_pass { call append_ua; }
+
     # Handle redirects, otherwise standard Vary handling code from previous
     # examples.
     sub vcl_fetch {



From lkarsten at varnish-cache.org  Thu Mar  8 13:30:54 2012
From: lkarsten at varnish-cache.org (Lasse Karstensen)
Date: Thu, 08 Mar 2012 14:30:54 +0100
Subject: [master] 4b6e84e Merge branch 'master' of
	/home/lkarsten/work/varnish-cache-fork
Message-ID: <E1S5dQk-0004rf-U8@project.varnish-software.com>

commit 4b6e84e491f2719436b4a8fffa100fe13f229668
Merge: fe828b4 72cb9f6
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Thu Mar 8 14:30:40 2012 +0100

    Merge branch 'master' of /home/lkarsten/work/varnish-cache-fork



From phk at varnish-cache.org  Sun Mar 11 07:48:24 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 11 Mar 2012 08:48:24 +0100
Subject: [master] e725920 Remove old Date: header before adding our new one.
Message-ID: <E1S6dVw-000425-Ly@project.varnish-software.com>

commit e725920a03e5329949635ced22d02f52ab91a5b9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Mar 9 09:58:41 2012 +0000

    Remove old Date: header before adding our new one.
    
    Submitted by:	scoof
    Fixes		#1104

diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 4760c12..1e065ba 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -128,6 +128,7 @@ RES_BuildHttp(const struct sess *sp)
 	if (req->res_mode & RES_CHUNKED)
 		http_SetHeader(req->resp, "Transfer-Encoding: chunked");
 
+	http_Unset(req->resp, H_Date);
 	VTIM_format(VTIM_real(), time_str);
 	http_PrintfHeader(req->resp, "Date: %s", time_str);
 



From phk at varnish-cache.org  Mon Mar 12 08:58:00 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 09:58:00 +0100
Subject: [master] 59c6ae4 Join struct vbo and struct busyobj
Message-ID: <E1S714q-0001BX-IS@project.varnish-software.com>

commit 59c6ae45f801a0f447d9f9647f2963950083c696
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 08:57:38 2012 +0000

    Join struct vbo and struct busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2bae6b4..48bdac0 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -108,7 +108,6 @@ struct poolparam;
 struct sess;
 struct sesspool;
 struct vbc;
-struct vbo;
 struct vef_priv;
 struct vrt_backend;
 struct vsb;
@@ -298,7 +297,7 @@ struct worker {
 	struct objhead		*nobjhead;
 	struct objcore		*nobjcore;
 	struct waitinglist	*nwaitinglist;
-	struct vbo		*nvbo;
+	struct busyobj		*nbo;
 	void			*nhashpriv;
 	struct dstat		stats;
 
@@ -459,7 +458,14 @@ enum busyobj_state_e {
 struct busyobj {
 	unsigned		magic;
 #define BUSYOBJ_MAGIC		0x23b95567
-	struct vbo		*vbo;
+	struct lock		mtx;
+	char			*end;
+
+	/*
+	 * All fields from refcount and down are zeroed when the busyobj
+	 * is recycled.
+	 */
+	unsigned		refcount;
 
 	uint8_t			*vary;
 	unsigned		is_gzip;
@@ -479,7 +485,7 @@ struct busyobj {
 	struct http_conn	htc;
 
 	enum body_status	body_status;
-	struct pool_task	task;
+	struct pool_task	fetch_task;
 
 	struct vef_priv		*vef_priv;
 
@@ -721,7 +727,7 @@ void VBO_Init(void);
 struct busyobj *VBO_GetBusyObj(struct worker *wrk);
 void VBO_RefBusyObj(const struct busyobj *busyobj);
 void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
-void VBO_Free(struct vbo **vbo);
+void VBO_Free(struct busyobj **vbo);
 
 /* cache_center.c [CNT] */
 void CNT_Session(struct sess *sp);
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 32ae7c8..601272f 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -40,15 +40,6 @@
 
 static struct mempool		*vbopool;
 
-struct vbo {
-	unsigned		magic;
-#define VBO_MAGIC		0xde3d8223
-	struct lock		mtx;
-	unsigned		refcount;
-	char			*end;
-	struct busyobj		bo;
-};
-
 /*--------------------------------------------------------------------
  */
 
@@ -56,7 +47,7 @@ void
 VBO_Init(void)
 {
 
-	vbopool = MPL_New("vbo", &cache_param->vbo_pool,
+	vbopool = MPL_New("busyobj", &cache_param->vbo_pool,
 	    &cache_param->workspace_backend);
 	AN(vbopool);
 }
@@ -65,88 +56,86 @@ VBO_Init(void)
  * BusyObj handling
  */
 
-static struct vbo *
+static struct busyobj *
 vbo_New(void)
 {
-	struct vbo *vbo;
+	struct busyobj *bo;
 	unsigned sz;
 
-	vbo = MPL_Get(vbopool, &sz);
-	AN(vbo);
-	vbo->magic = VBO_MAGIC;
-	vbo->end = (char *)vbo + sz;
-	Lck_New(&vbo->mtx, lck_busyobj);
-	return (vbo);
+	bo = MPL_Get(vbopool, &sz);
+	XXXAN(bo);
+	bo->magic = BUSYOBJ_MAGIC;
+	bo->end = (char *)bo + sz;
+	Lck_New(&bo->mtx, lck_busyobj);
+	return (bo);
 }
 
 void
-VBO_Free(struct vbo **vbop)
+VBO_Free(struct busyobj **bop)
 {
-	struct vbo *vbo;
+	struct busyobj *bo;
 
-	AN(vbop);
-	vbo = *vbop;
-	*vbop = NULL;
-	CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC);
-	AZ(vbo->refcount);
-	Lck_Delete(&vbo->mtx);
-	MPL_Free(vbopool, vbo);
+	AN(bop);
+	bo = *bop;
+	*bop = NULL;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	AZ(bo->refcount);
+	Lck_Delete(&bo->mtx);
+	MPL_Free(vbopool, bo);
 }
 
 struct busyobj *
 VBO_GetBusyObj(struct worker *wrk)
 {
-	struct vbo *vbo = NULL;
+	struct busyobj *bo = NULL;
 	uint16_t nhttp;
 	unsigned sz;
 	char *p;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	if (wrk->nvbo != NULL) {
-		vbo = wrk->nvbo;
-		wrk->nvbo = NULL;
+	if (wrk->nbo != NULL) {
+		bo = wrk->nbo;
+		wrk->nbo = NULL;
 	}
 
-	if (vbo == NULL)
-		vbo = vbo_New();
+	if (bo == NULL)
+		bo = vbo_New();
 
-	CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC);
-	AZ(vbo->refcount);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	AZ(bo->refcount);
 
-	AZ(vbo->bo.magic);
-	vbo->refcount = 1;
-	vbo->bo.magic = BUSYOBJ_MAGIC;
-	vbo->bo.vbo = vbo;
+	bo->refcount = 1;
 
-	p = (void*)(vbo + 1);
+	p = (void*)(bo + 1);
 	p = (void*)PRNDUP(p);
-	assert(p < vbo->end);
+	assert(p < bo->end);
 
 	nhttp = (uint16_t)cache_param->http_max_hdr;
 	sz = HTTP_estimate(nhttp);
 
-	vbo->bo.bereq = HTTP_create(p, nhttp);
+	bo->bereq = HTTP_create(p, nhttp);
 	p += sz;
 	p = (void*)PRNDUP(p);
-	assert(p < vbo->end);
+	assert(p < bo->end);
 
-	vbo->bo.beresp = HTTP_create(p, nhttp);
+	bo->beresp = HTTP_create(p, nhttp);
 	p += sz;
 	p = (void*)PRNDUP(p);
-	assert(p < vbo->end);
+	assert(p < bo->end);
 
 	sz = cache_param->vsl_buffer;
-	VSL_Setup(vbo->bo.vsl, p, sz);
+	VSL_Setup(bo->vsl, p, sz);
 	p += sz;
 	p = (void*)PRNDUP(p);
-	assert(p < vbo->end);
+	assert(p < bo->end);
 
-	WS_Init(vbo->bo.ws, "bo", p, vbo->end - p);
+	WS_Init(bo->ws, "bo", p, bo->end - p);
 
-	return (&vbo->bo);
+	return (bo);
 }
 
+#if 0
 void
 VBO_RefBusyObj(const struct busyobj *busyobj)
 {
@@ -160,12 +149,12 @@ VBO_RefBusyObj(const struct busyobj *busyobj)
 	vbo->refcount++;
 	Lck_Unlock(&vbo->mtx);
 }
+#endif
 
 void
 VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 {
 	struct busyobj *bo;
-	struct vbo *vbo;
 	unsigned r;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -173,22 +162,21 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 	bo = *pbo;
 	*pbo = NULL;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vbo = bo->vbo;
-	CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC);
-	Lck_Lock(&vbo->mtx);
-	assert(vbo->refcount > 0);
-	r = --vbo->refcount;
-	Lck_Unlock(&vbo->mtx);
+	Lck_Lock(&bo->mtx);
+	assert(bo->refcount > 0);
+	r = --bo->refcount;
+	Lck_Unlock(&bo->mtx);
 
 	if (r)
 		return;
 
-	VSL_Flush(vbo->bo.vsl, 0);
-	/* XXX: Sanity checks & cleanup */
-	memset(&vbo->bo, 0, sizeof vbo->bo);
+	VSL_Flush(bo->vsl, 0);
+
+	memset(&bo->refcount, 0,
+	    sizeof *bo - offsetof(struct busyobj, refcount));
 
-	if (cache_param->bo_cache && wrk->nvbo == NULL)
-		wrk->nvbo = vbo;
+	if (cache_param->bo_cache && wrk->nbo == NULL)
+		wrk->nbo = bo;
 	else
-		VBO_Free(&vbo);
+		VBO_Free(&bo);
 }
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index ecea4b5..1ae5dd7 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -906,9 +906,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 #if 1
 	FetchBody(wrk, bo);
 #else
-	bo->task.func = FetchBody;
-	bo->task.priv = bo;
-	if (Pool_Task(wrk->pool, &bo->task, POOL_NO_QUEUE)) {
+	bo->fetch_task.func = FetchBody;
+	bo->fetch_task.priv = bo;
+	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) {
 		FetchBody(wrk, bo);
 	} else {
 		while (bo->state < BOS_FAILED)
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 1e21214..951cef9 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -131,8 +131,6 @@ HSH_Cleanup(struct worker *wrk)
 		free(wrk->nhashpriv);
 		wrk->nhashpriv = NULL;
 	}
-	if (wrk->nvbo != NULL)
-		VBO_Free(&wrk->nvbo);
 }
 
 void
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index cc3a061..7b65479 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -147,8 +147,8 @@ wrk_thread_real(void *priv, unsigned thread_workspace)
 	if (w->vcl != NULL)
 		VCL_Rel(&w->vcl);
 	AZ(pthread_cond_destroy(&w->cond));
-	if (w->nvbo != NULL)
-		VBO_Free(&w->nvbo);
+	if (w->nbo != NULL)
+		VBO_Free(&w->nbo);
 	HSH_Cleanup(w);
 	WRK_SumStat(w);
 	return (NULL);



From phk at varnish-cache.org  Mon Mar 12 09:32:17 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 10:32:17 +0100
Subject: [master] d1d7134 Always try to set another thread on the body-fetch
	job.
Message-ID: <E1S71c1-0004OK-SR@project.varnish-software.com>

commit d1d71344524068ad56d170bd29feb719d9a9b902
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 09:31:40 2012 +0000

    Always try to set another thread on the body-fetch job.
    
    Slight rearrangement of FetchBody()

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 1ae5dd7..7df47bb 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -903,18 +903,13 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	bo = req->busyobj;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-#if 1
-	FetchBody(wrk, bo);
-#else
 	bo->fetch_task.func = FetchBody;
 	bo->fetch_task.priv = bo;
-	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE)) {
+	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
 		FetchBody(wrk, bo);
-	} else {
-		while (bo->state < BOS_FAILED)
-			(void)usleep(10000);
-	}
-#endif
+
+	while (bo->state < BOS_FAILED)
+		(void)usleep(10000);
 	assert(bo->state >= BOS_FAILED);
 
 	http_Teardown(bo->bereq);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 88bca02..e3bf6dc 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -627,44 +627,43 @@ FetchBody(struct worker *wrk, void *priv)
 		wrk->stats.fetch_failed++;
 		VDI_CloseFd(&bo->vbc);
 		obj->len = 0;
-		bo->stats = NULL;
-		return;
-	}
-
-	assert(bo->state == BOS_FETCHING);
+	} else {
+		assert(bo->state == BOS_FETCHING);
 
-	if (cls == 0 && bo->should_close)
-		cls = 1;
+		if (cls == 0 && bo->should_close)
+			cls = 1;
 
-	VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
+		VSLb(bo->vsl, SLT_Length, "%zd", obj->len);
 
-	{
-	/* Sanity check fetch methods accounting */
-		ssize_t uu;
+		{
+		/* Sanity check fetch methods accounting */
+			ssize_t uu;
 
-		uu = 0;
-		VTAILQ_FOREACH(st, &obj->store, list)
-			uu += st->len;
-		if (bo->do_stream)
-			/* Streaming might have started freeing stuff */
-			assert (uu <= obj->len);
+			uu = 0;
+			VTAILQ_FOREACH(st, &obj->store, list)
+				uu += st->len;
+			if (bo->do_stream)
+				/* Streaming might have started freeing stuff */
+				assert (uu <= obj->len);
 
-		else
-			assert(uu == obj->len);
-	}
+			else
+				assert(uu == obj->len);
+		}
 
-	if (mklen > 0) {
-		http_Unset(obj->http, H_Content_Length);
-		http_PrintfHeader(obj->http, "Content-Length: %zd", obj->len);
-	}
+		if (mklen > 0) {
+			http_Unset(obj->http, H_Content_Length);
+			http_PrintfHeader(obj->http,
+			    "Content-Length: %zd", obj->len);
+		}
 
-	bo->state = BOS_FINISHED;
+		bo->state = BOS_FINISHED;
 
-	if (cls)
-		VDI_CloseFd(&bo->vbc);
-	else
-		VDI_RecycleFd(&bo->vbc);
+		if (cls)
+			VDI_CloseFd(&bo->vbc);
+		else
+			VDI_RecycleFd(&bo->vbc);
 
+	}
 	bo->stats = NULL;
 }
 



From apj at varnish-cache.org  Mon Mar 12 11:25:51 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Mon, 12 Mar 2012 12:25:51 +0100
Subject: [master] c5e141b Correct function name
Message-ID: <E1S73Nv-0003yh-9k@project.varnish-software.com>

commit c5e141be4c608fc7264d36506fd79c49e04a86a5
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Mar 12 12:23:47 2012 +0100

    Correct function name

diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst
index 9c28a3c..a1ece16 100644
--- a/doc/sphinx/reference/vmod_std.rst
+++ b/doc/sphinx/reference/vmod_std.rst
@@ -50,7 +50,7 @@ Description
 Example
 	set beresp.http.x-nice = std.tolower("VerY");
 
-set_up_tos
+set_ip_tos
 ----------
 Prototype
 	set_ip_tos(INT i)



From apj at varnish-cache.org  Mon Mar 12 11:25:51 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Mon, 12 Mar 2012 12:25:51 +0100
Subject: [master] f740e80 Small doc fixes
Message-ID: <E1S73Nv-0003ym-Ay@project.varnish-software.com>

commit f740e80841888b03cc5685dcfeb3974afe2c376c
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Mar 12 12:25:41 2012 +0100

    Small doc fixes

diff --git a/doc/sphinx/tutorial/logging.rst b/doc/sphinx/tutorial/logging.rst
index 68f1d87..1f0bc18 100644
--- a/doc/sphinx/tutorial/logging.rst
+++ b/doc/sphinx/tutorial/logging.rst
@@ -9,7 +9,7 @@ memory segment. When the end of the segment is reached we start over,
 overwriting old data. This is much, much faster then logging to a file
 and it doesn't require disk space.
 
-The flip side is that if you forget to have program actually write the
+The flip side is that if you forget to have a program actually write the
 logs to disk they will disappear.
 
 varnishlog is one of the programs you can use to look at what Varnish
diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst
index 7171ab6..54bce37 100644
--- a/doc/sphinx/tutorial/vcl.rst
+++ b/doc/sphinx/tutorial/vcl.rst
@@ -135,7 +135,7 @@ down for, uhm, examples.
 Example 1 - manipulating headers
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Lets say we want to remove the cookie for all objects in the /static
+Lets say we want to remove the cookie for all objects in the /images
 directory of our web server::
 
   sub vcl_recv {



From phk at varnish-cache.org  Mon Mar 12 12:28:07 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 13:28:07 +0100
Subject: [master] b8d4eb8 Hand FetchBody() its own busyobj->refcount,
	and have it release it when done.
Message-ID: <E1S74MB-0001qb-W6@project.varnish-software.com>

commit b8d4eb8a23d1971aaef3fe087d024b7f1dd78df9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 12:27:34 2012 +0000

    Hand FetchBody() its own busyobj->refcount, and have it release it
    when done.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 48bdac0..99c66c3 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -725,7 +725,7 @@ double BAN_Time(const struct ban *ban);
 /* cache_busyobj.c */
 void VBO_Init(void);
 struct busyobj *VBO_GetBusyObj(struct worker *wrk);
-void VBO_RefBusyObj(const struct busyobj *busyobj);
+void VBO_RefBusyObj(struct busyobj *busyobj);
 void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
 void VBO_Free(struct busyobj **vbo);
 
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 601272f..8de928a 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -135,21 +135,16 @@ VBO_GetBusyObj(struct worker *wrk)
 	return (bo);
 }
 
-#if 0
 void
-VBO_RefBusyObj(const struct busyobj *busyobj)
+VBO_RefBusyObj(struct busyobj *bo)
 {
-	struct vbo *vbo;
-
-	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
-	vbo = busyobj->vbo;
-	CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC);
-	Lck_Lock(&vbo->mtx);
-	assert(vbo->refcount > 0);
-	vbo->refcount++;
-	Lck_Unlock(&vbo->mtx);
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	Lck_Lock(&bo->mtx);
+	assert(bo->refcount > 0);
+	bo->refcount++;
+	Lck_Unlock(&bo->mtx);
 }
-#endif
 
 void
 VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
@@ -157,7 +152,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 	struct busyobj *bo;
 	unsigned r;
 
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
 	AN(pbo);
 	bo = *pbo;
 	*pbo = NULL;
@@ -175,7 +170,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 	memset(&bo->refcount, 0,
 	    sizeof *bo - offsetof(struct busyobj, refcount));
 
-	if (cache_param->bo_cache && wrk->nbo == NULL)
+	if (cache_param->bo_cache && wrk != NULL && wrk->nbo == NULL)
 		wrk->nbo = bo;
 	else
 		VBO_Free(&bo);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 7df47bb..aa07828 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -905,6 +905,10 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 
 	bo->fetch_task.func = FetchBody;
 	bo->fetch_task.priv = bo;
+
+	/* Gain a reference for FetchBody() */
+	VBO_RefBusyObj(bo);
+
 	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
 		FetchBody(wrk, bo);
 
@@ -912,8 +916,6 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 		(void)usleep(10000);
 	assert(bo->state >= BOS_FAILED);
 
-	http_Teardown(bo->bereq);
-	http_Teardown(bo->beresp);
 	bo->vfp = NULL;
 	assert(WRW_IsReleased(wrk));
 	AZ(bo->vbc);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index e3bf6dc..a52201a 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -298,7 +298,7 @@ fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 	do {
 		/* Skip leading whitespace */
 		do {
-			if (HTC_Read(htc, buf, 1) <= 0) 
+			if (HTC_Read(htc, buf, 1) <= 0)
 				return (FetchError(bo, "chunked read err"));
 		} while (vct_islws(buf[0]));
 
@@ -527,7 +527,13 @@ FetchHdr(struct sess *sp, int need_host_hdr, int sendbody)
 	return (0);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * This function is either called by the requesting thread OR by a
+ * dedicated body-fetch work-thread.
+ *
+ * We get passed the busyobj in the priv arg, and we inherit a
+ * refcount on it, which we must release, when done fetching.
+ */
 
 void
 FetchBody(struct worker *wrk, void *priv)
@@ -623,6 +629,9 @@ FetchBody(struct worker *wrk, void *priv)
 	    bo->body_status, body_status(bo->body_status),
 	    cls, mklen);
 
+	http_Teardown(bo->bereq);
+	http_Teardown(bo->beresp);
+
 	if (bo->state == BOS_FAILED) {
 		wrk->stats.fetch_failed++;
 		VDI_CloseFd(&bo->vbc);
@@ -665,6 +674,7 @@ FetchBody(struct worker *wrk, void *priv)
 
 	}
 	bo->stats = NULL;
+	VBO_DerefBusyObj(NULL, &bo);
 }
 
 /*--------------------------------------------------------------------



From martin at varnish-cache.org  Mon Mar 12 13:24:36 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Mon, 12 Mar 2012 14:24:36 +0100
Subject: [master] b78280f Decrement the n_waitinglist counter when freeing
	waitinglists.
Message-ID: <E1S75Eq-0004ez-74@project.varnish-software.com>

commit b78280f445ed69af77b42154dff4e6d3a70e0b5f
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 12 12:39:36 2012 +0100

    Decrement the n_waitinglist counter when freeing waitinglists.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index aa07828..1c9d48b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -935,7 +935,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 		AN(req->obj->objcore);
 		AN(req->obj->objcore->ban);
 		AZ(req->obj->ws_o->overflow);
-		HSH_Unbusy(req->obj->objcore);
+		HSH_Unbusy(&wrk->stats, req->obj->objcore);
 	}
 	VBO_DerefBusyObj(wrk, &req->busyobj);
 	wrk->acct_tmp.fetch++;
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 951cef9..6e1c27f 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -125,6 +125,7 @@ HSH_Cleanup(struct worker *wrk)
 	if (wrk->nwaitinglist != NULL) {
 		FREE_OBJ(wrk->nwaitinglist);
 		wrk->nwaitinglist = NULL;
+		wrk->stats.n_waitinglist--;
 	}
 	if (wrk->nhashpriv != NULL) {
 		/* XXX: If needed, add slinger method for this */
@@ -468,7 +469,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
  */
 
 static void
-hsh_rush(struct objhead *oh)
+hsh_rush(struct dstat *ds, struct objhead *oh)
 {
 	unsigned u;
 	struct sess *sp;
@@ -497,6 +498,7 @@ hsh_rush(struct objhead *oh)
 	if (VTAILQ_EMPTY(&wl->list)) {
 		oh->waitinglist = NULL;
 		FREE_OBJ(wl);
+		ds->n_waitinglist--;
 	}
 }
 
@@ -576,12 +578,12 @@ HSH_Drop(struct worker *wrk, struct object **oo)
 	AssertObjCorePassOrBusy((*oo)->objcore);
 	(*oo)->exp.ttl = -1.;
 	if ((*oo)->objcore != NULL)		/* Pass has no objcore */
-		HSH_Unbusy((*oo)->objcore);
+		HSH_Unbusy(&wrk->stats, (*oo)->objcore);
 	(void)HSH_Deref(&wrk->stats, NULL, oo);
 }
 
 void
-HSH_Unbusy(struct objcore *oc)
+HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 {
 	struct objhead *oh;
 
@@ -603,7 +605,7 @@ HSH_Unbusy(struct objcore *oc)
 	oc->flags &= ~OC_F_BUSY;
 	oc->busyobj = NULL;
 	if (oh->waitinglist != NULL)
-		hsh_rush(oh);
+		hsh_rush(ds, oh);
 	AN(oc->ban);
 	Lck_Unlock(&oh->mtx);
 }
@@ -680,7 +682,7 @@ HSH_Deref(struct dstat *ds, struct objcore *oc, struct object **oo)
 		AN(oc->methods);
 	}
 	if (oh->waitinglist != NULL)
-		hsh_rush(oh);
+		hsh_rush(ds, oh);
 	Lck_Unlock(&oh->mtx);
 	if (r != 0)
 		return (r);
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index b15d9df..a3577a7 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -53,7 +53,6 @@ struct hash_slinger {
 /* cache_hash.c */
 void HSH_Cleanup(struct worker *w);
 struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh);
-void HSH_Unbusy(struct objcore *);
 void HSH_Ref(struct objcore *o);
 void HSH_Drop(struct worker *, struct object **);
 void HSH_Init(const struct hash_slinger *slinger);
@@ -94,6 +93,7 @@ struct objhead {
 #define hoh_head _u.n.u_n_hoh_head
 };
 
+void HSH_Unbusy(struct dstat *, struct objcore *);
 void HSH_DeleteObjHead(struct dstat *, struct objhead *oh);
 int HSH_Deref(struct dstat *, struct objcore *oc, struct object **o);
 #endif /* VARNISH_CACHE_CHILD */



From phk at varnish-cache.org  Mon Mar 12 13:50:01 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 14:50:01 +0100
Subject: [master] 40d62bd Have FetchBody reset bo->vfp once it's done with it.
Message-ID: <E1S75dR-0002t4-99@project.varnish-software.com>

commit 40d62bd804931d5bc3a680768309529548c5d00e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 13:49:43 2012 +0000

    Have FetchBody reset bo->vfp once it's done with it.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 1c9d48b..3e80d06 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -916,7 +916,6 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 		(void)usleep(10000);
 	assert(bo->state >= BOS_FAILED);
 
-	bo->vfp = NULL;
 	assert(WRW_IsReleased(wrk));
 	AZ(bo->vbc);
 	AN(req->director);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index a52201a..172f422 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -625,6 +625,8 @@ FetchBody(struct worker *wrk, void *priv)
 	 */
 	AZ(vfp_nop_end(bo));
 
+	bo->vfp = NULL;
+
 	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s) cls %d mklen %d",
 	    bo->body_status, body_status(bo->body_status),
 	    cls, mklen);
@@ -674,7 +676,7 @@ FetchBody(struct worker *wrk, void *priv)
 
 	}
 	bo->stats = NULL;
-	VBO_DerefBusyObj(NULL, &bo);
+	VBO_DerefBusyObj(wrk, &bo);
 }
 
 /*--------------------------------------------------------------------



From phk at varnish-cache.org  Mon Mar 12 14:58:27 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 15:58:27 +0100
Subject: [master] 7b25530 These testcases depend on us not streaming,
	so they get a 503 for the failure to fetch a body. Make them
	explicitly disable streaming.
Message-ID: <E1S76hf-0004UW-KG@project.varnish-software.com>

commit 7b2553097060e4e01cbf42cbcc3951a28597bad9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 14:57:49 2012 +0000

    These testcases depend on us not streaming, so they get a 503
    for the failure to fetch a body.  Make them explicitly disable
    streaming.

diff --git a/bin/varnishtest/tests/b00020.vtc b/bin/varnishtest/tests/b00020.vtc
index d937ac0..365c305 100644
--- a/bin/varnishtest/tests/b00020.vtc
+++ b/bin/varnishtest/tests/b00020.vtc
@@ -9,7 +9,11 @@ server s1 {
 	send "Baba\n"
 } -start
 
-varnish v1 -vcl+backend {} -start
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_stream = false;
+	}
+} -start
 varnish v1 -cliok "param.set between_bytes_timeout 1"
 
 client c1 {
diff --git a/bin/varnishtest/tests/b00021.vtc b/bin/varnishtest/tests/b00021.vtc
index d46caa2..5c3261a 100644
--- a/bin/varnishtest/tests/b00021.vtc
+++ b/bin/varnishtest/tests/b00021.vtc
@@ -13,6 +13,9 @@ varnish v1 -vcl+backend {
 	sub vcl_miss {
 		set bereq.between_bytes_timeout = 2s;
 	}
+	sub vcl_fetch {
+		set beresp.do_stream = false;
+	}
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/b00022.vtc b/bin/varnishtest/tests/b00022.vtc
index aa625b6..722d9f1 100644
--- a/bin/varnishtest/tests/b00022.vtc
+++ b/bin/varnishtest/tests/b00022.vtc
@@ -15,6 +15,9 @@ varnish v1 -vcl {
 		.port = "${s1_port}";
 		.between_bytes_timeout = 1s;
 	}
+	sub vcl_fetch {
+		set beresp.do_stream = false;
+	}
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/b00027.vtc b/bin/varnishtest/tests/b00027.vtc
index 4076019..9cc7a7a 100644
--- a/bin/varnishtest/tests/b00027.vtc
+++ b/bin/varnishtest/tests/b00027.vtc
@@ -11,7 +11,11 @@ server s1 {
 	send "\n"
 } -start
 
-varnish v1 -vcl+backend {} -start
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_stream = false;
+	}
+} -start
 
 client c1 {
 	txreq -url /foo
diff --git a/bin/varnishtest/tests/g00004.vtc b/bin/varnishtest/tests/g00004.vtc
index 5d518e0..0ab74ad 100644
--- a/bin/varnishtest/tests/g00004.vtc
+++ b/bin/varnishtest/tests/g00004.vtc
@@ -21,6 +21,8 @@ varnish v1 \
 	-arg {-p diag_bitmap=0x00010000} \
 	-vcl+backend {
 	sub vcl_fetch {
+		set beresp.do_stream = false;
+
 		if (req.url == "/gunzip") {
 			set beresp.do_gunzip = true;
 		} 
diff --git a/bin/varnishtest/tests/r00387.vtc b/bin/varnishtest/tests/r00387.vtc
index fb73e6b..d4a32f9 100644
--- a/bin/varnishtest/tests/r00387.vtc
+++ b/bin/varnishtest/tests/r00387.vtc
@@ -11,7 +11,11 @@ server s1 {
 	send "\r\n"
 } -start
 
-varnish v1 -vcl+backend {} -start
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_stream = false;
+	}
+} -start
 
 client c1 {
 	txreq
diff --git a/bin/varnishtest/tests/r00942.vtc b/bin/varnishtest/tests/r00942.vtc
index 0df8eb2..6bb31e1 100644
--- a/bin/varnishtest/tests/r00942.vtc
+++ b/bin/varnishtest/tests/r00942.vtc
@@ -26,7 +26,11 @@ server s1 {
 
 varnish v1 \
 	-arg {-p diag_bitmap=0x00010000} \
-	-vcl+backend {} 
+	-vcl+backend {
+		sub vcl_fetch {
+			set beresp.do_stream = false;
+		}
+} 
 
 varnish v1 -start
 
diff --git a/bin/varnishtest/tests/r01036.vtc b/bin/varnishtest/tests/r01036.vtc
index 8fb7600..1c77919 100644
--- a/bin/varnishtest/tests/r01036.vtc
+++ b/bin/varnishtest/tests/r01036.vtc
@@ -9,6 +9,7 @@ server s1 {
 
 varnish v1 -arg "-smalloc,1M" -arg "-pgzip_level=0" -vcl+backend {
 	sub vcl_fetch {
+		set beresp.do_stream = false;
 		set beresp.do_gzip = true;
 	}
 } -start



From lkarsten at varnish-cache.org  Mon Mar 12 15:13:27 2012
From: lkarsten at varnish-cache.org (Lasse Karstensen)
Date: Mon, 12 Mar 2012 16:13:27 +0100
Subject: [master] d96e763 Include device class when hashing
Message-ID: <E1S76wB-00076e-NC@project.varnish-software.com>

commit d96e763c12f2d7229fd642e6e4900fb48fd37586
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Mon Mar 12 16:13:56 2012 +0100

    Include device class when hashing

diff --git a/doc/sphinx/tutorial/devicedetection.rst b/doc/sphinx/tutorial/devicedetection.rst
index abf729e..9f02493 100644
--- a/doc/sphinx/tutorial/devicedetection.rst
+++ b/doc/sphinx/tutorial/devicedetection.rst
@@ -228,6 +228,11 @@ special needs in VCL, you can use the X-UA-Device header like this::
             set req.backend = mobile;
         }
     }
+    sub vcl_hash {
+        if (req.http.X-UA-Device) {
+            hash_data(req.http.X-UA-Device);
+        }
+    }
 
 Redirecting mobile clients
 --------------------------



From lkarsten at varnish-cache.org  Mon Mar 12 15:13:27 2012
From: lkarsten at varnish-cache.org (Lasse Karstensen)
Date: Mon, 12 Mar 2012 16:13:27 +0100
Subject: [master] 9831187 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1S76wB-00076j-O9@project.varnish-software.com>

commit 9831187e28c682075897cd1b4295898e2f9c170d
Merge: d96e763 7b25530
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Mon Mar 12 16:14:17 2012 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From phk at varnish-cache.org  Mon Mar 12 15:42:19 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 16:42:19 +0100
Subject: [master] 0326c20 Quick polish over HSH_Lookup():
Message-ID: <E1S77O7-00018H-19@project.varnish-software.com>

commit 0326c20463a90e3750dbcc492244da1d0f03d9af
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 15:41:07 2012 +0000

    Quick polish over HSH_Lookup():
    
    Now that we have busyobjs, we do not need to return the objhdr
    separately.
    
    Introduce a local req variable to reduce sp->req dereferences.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 3e80d06..4d18395 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1104,9 +1104,8 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 	req->vary_e = (void*)req->ws->r;
 	req->vary_b[2] = '\0';
 
-	oc = HSH_Lookup(sp, &oh);
 	AZ(req->objcore);
-
+	oc = HSH_Lookup(sp);
 	if (oc == NULL) {
 		/*
 		 * We lost the session to a busy object, disembark the
@@ -1117,8 +1116,10 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 		 */
 		return (1);
 	}
+	AZ(req->objcore);
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	oh = oc->objhead;
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
 	/* If we inserted a new object it's a miss */
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 6e1c27f..55215e6 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -282,36 +282,38 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc)
  */
 
 struct objcore *
-HSH_Lookup(struct sess *sp, struct objhead **poh)
+HSH_Lookup(struct sess *sp)
 {
 	struct worker *wrk;
 	struct objhead *oh;
 	struct objcore *oc;
 	struct objcore *busy_oc, *grace_oc;
 	struct object *o;
+	struct req *req;
 	double grace_ttl;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->req->http, HTTP_MAGIC);
-	AN(sp->req->director);
-	AN(hash);
 	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	req = sp->req;
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->http, HTTP_MAGIC);
+	AN(req->director);
+	AN(hash);
 
 	hsh_prealloc(wrk);
-	memcpy(sp->wrk->nobjhead->digest, sp->req->digest,
-	    sizeof sp->req->digest);
+	memcpy(sp->wrk->nobjhead->digest, req->digest, sizeof req->digest);
 	if (cache_param->diag_bitmap & 0x80000000)
 		hsh_testmagic(sp->wrk->nobjhead->digest);
 
-	if (sp->req->hash_objhead != NULL) {
+	if (req->hash_objhead != NULL) {
 		/*
 		 * This sess came off the waiting list, and brings a
 		 * oh refcnt with it.
 		 */
-		CHECK_OBJ_NOTNULL(sp->req->hash_objhead, OBJHEAD_MAGIC);
-		oh = sp->req->hash_objhead;
-		sp->req->hash_objhead = NULL;
+		CHECK_OBJ_NOTNULL(req->hash_objhead, OBJHEAD_MAGIC);
+		oh = req->hash_objhead;
+		req->hash_objhead = NULL;
 	} else {
 		AN(wrk->nobjhead);
 		oh = hash->lookup(wrk, wrk->nobjhead);
@@ -333,7 +335,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 
 		if (oc->flags & OC_F_BUSY) {
 			CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
-			if (sp->req->hash_ignore_busy)
+			if (req->hash_ignore_busy)
 				continue;
 
 			if (oc->busyobj->vary != NULL &&
@@ -381,20 +383,20 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	 * XXX: serialize fetch of all Vary's if grace is possible.
 	 */
 
-	AZ(sp->req->objcore);
-	sp->req->objcore = grace_oc;		/* XXX: Hack-ish */
+	AZ(req->objcore);
+	req->objcore = grace_oc;		/* XXX: Hack-ish */
 	if (oc == NULL			/* We found no live object */
 	    && grace_oc != NULL		/* There is a grace candidate */
 	    && (busy_oc != NULL		/* Somebody else is already busy */
-	    || !VDI_Healthy(sp->req->director, sp))) {
+	    || !VDI_Healthy(req->director, sp))) {
 					/* Or it is impossible to fetch */
 		o = oc_getobj(&sp->wrk->stats, grace_oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		oc = grace_oc;
 	}
-	sp->req->objcore = NULL;
+	req->objcore = NULL;
 
-	if (oc != NULL && !sp->req->hash_always_miss) {
+	if (oc != NULL && !req->hash_always_miss) {
 		o = oc_getobj(&sp->wrk->stats, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		assert(oc->objhead == oh);
@@ -406,13 +408,12 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		assert(oh->refcnt > 1);
 		Lck_Unlock(&oh->mtx);
 		assert(hash->deref(oh));
-		*poh = oh;
 		return (oc);
 	}
 
 	if (busy_oc != NULL) {
 		/* There are one or more busy objects, wait for them */
-		if (sp->req->esi_level == 0) {
+		if (req->esi_level == 0) {
 			CHECK_OBJ_NOTNULL(sp->wrk->nwaitinglist,
 			    WAITINGLIST_MAGIC);
 			if (oh->waitinglist == NULL) {
@@ -422,7 +423,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 			VTAILQ_INSERT_TAIL(&oh->waitinglist->list, sp, list);
 		}
 		if (cache_param->diag_bitmap & 0x20)
-			VSLb(sp->req->vsl, SLT_Debug,
+			VSLb(req->vsl, SLT_Debug,
 				"on waiting list <%p>", oh);
 		SES_Charge(sp);
 		/*
@@ -430,7 +431,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		 * back when the sess comes off the waiting list and
 		 * calls us again
 		 */
-		sp->req->hash_objhead = oh;
+		req->hash_objhead = oh;
 		sp->wrk = NULL;
 		Lck_Unlock(&oh->mtx);
 		return (NULL);
@@ -442,16 +443,16 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	AN(oc->flags & OC_F_BUSY);
 	oc->refcnt = 1;
 
-	AZ(sp->req->busyobj);
-	sp->req->busyobj = VBO_GetBusyObj(wrk);
-	sp->req->busyobj->vsl->wid = sp->vsl_id;
+	AZ(req->busyobj);
+	req->busyobj = VBO_GetBusyObj(wrk);
+	req->busyobj->vsl->wid = sp->vsl_id;
 
-	VRY_Validate(sp->req->vary_b);
-	if (sp->req->vary_l != NULL)
-		sp->req->busyobj->vary = sp->req->vary_b;
+	VRY_Validate(req->vary_b);
+	if (req->vary_l != NULL)
+		req->busyobj->vary = req->vary_b;
 	else
-		sp->req->busyobj->vary = NULL;
-	oc->busyobj = sp->req->busyobj;
+		req->busyobj->vary = NULL;
+	oc->busyobj = req->busyobj;
 
 	/*
 	 * Busy objects go on the tail, so they will not trip up searches.
@@ -461,7 +462,6 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	oc->objhead = oh;
 	/* NB: do not deref objhead the new object inherits our reference */
 	Lck_Unlock(&oh->mtx);
-	*poh = oh;
 	return (oc);
 }
 
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index a3577a7..4935bfb 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -52,7 +52,7 @@ struct hash_slinger {
 
 /* cache_hash.c */
 void HSH_Cleanup(struct worker *w);
-struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh);
+struct objcore *HSH_Lookup(struct sess *sp);
 void HSH_Ref(struct objcore *o);
 void HSH_Drop(struct worker *, struct object **);
 void HSH_Init(const struct hash_slinger *slinger);



From phk at varnish-cache.org  Mon Mar 12 15:42:19 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 16:42:19 +0100
Subject: [master] aea8559 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1S77O7-00018N-2B@project.varnish-software.com>

commit aea855912ec8e140435e7d5f6604e1c6062faf79
Merge: 0326c20 9831187
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 15:42:10 2012 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From phk at varnish-cache.org  Mon Mar 12 15:49:15 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 16:49:15 +0100
Subject: [master] e556146 More deref-reduction:
Message-ID: <E1S77Up-0002ST-Ai@project.varnish-software.com>

commit e556146d8bca01cf714a0594e4454d725a2154c9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 15:48:43 2012 +0000

    More deref-reduction:
    
    VRY only needs req, not sess.
    Overlooked some lingering sp->wrk's in HSH_Lookup()

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 99c66c3..6aceab0 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -954,8 +954,8 @@ void RES_BuildHttp(const struct sess *sp);
 void RES_WriteObj(struct sess *sp);
 
 /* cache_vary.c */
-struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);
-int VRY_Match(const struct sess *sp, const uint8_t *vary);
+struct vsb *VRY_Create(struct req *sp, const struct http *hp);
+int VRY_Match(struct req *, const uint8_t *vary);
 void VRY_Validate(const uint8_t *vary);
 
 /* cache_vcl.c */
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 4d18395..1192f81 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -784,7 +784,7 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req)
 	/* Create Vary instructions */
 	if (req->objcore != NULL) {
 		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-		vary = VRY_Create(sp, bo->beresp);
+		vary = VRY_Create(req, bo->beresp);
 		if (vary != NULL) {
 			varyl = VSB_len(vary);
 			assert(varyl > 0);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 55215e6..34e86a4 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -302,9 +302,9 @@ HSH_Lookup(struct sess *sp)
 	AN(hash);
 
 	hsh_prealloc(wrk);
-	memcpy(sp->wrk->nobjhead->digest, req->digest, sizeof req->digest);
+	memcpy(wrk->nobjhead->digest, req->digest, sizeof req->digest);
 	if (cache_param->diag_bitmap & 0x80000000)
-		hsh_testmagic(sp->wrk->nobjhead->digest);
+		hsh_testmagic(wrk->nobjhead->digest);
 
 	if (req->hash_objhead != NULL) {
 		/*
@@ -339,21 +339,21 @@ HSH_Lookup(struct sess *sp)
 				continue;
 
 			if (oc->busyobj->vary != NULL &&
-			    !VRY_Match(sp, oc->busyobj->vary))
+			    !VRY_Match(req, oc->busyobj->vary))
 				continue;
 
 			busy_oc = oc;
 			continue;
 		}
 
-		o = oc_getobj(&sp->wrk->stats, oc);
+		o = oc_getobj(&wrk->stats, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 
 		if (o->exp.ttl <= 0.)
 			continue;
 		if (BAN_CheckObject(o, sp))
 			continue;
-		if (o->vary != NULL && !VRY_Match(sp, o->vary))
+		if (o->vary != NULL && !VRY_Match(req, o->vary))
 			continue;
 
 		/* If still valid, use it */
@@ -390,14 +390,14 @@ HSH_Lookup(struct sess *sp)
 	    && (busy_oc != NULL		/* Somebody else is already busy */
 	    || !VDI_Healthy(req->director, sp))) {
 					/* Or it is impossible to fetch */
-		o = oc_getobj(&sp->wrk->stats, grace_oc);
+		o = oc_getobj(&wrk->stats, grace_oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		oc = grace_oc;
 	}
 	req->objcore = NULL;
 
 	if (oc != NULL && !req->hash_always_miss) {
-		o = oc_getobj(&sp->wrk->stats, oc);
+		o = oc_getobj(&wrk->stats, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		assert(oc->objhead == oh);
 
@@ -414,11 +414,11 @@ HSH_Lookup(struct sess *sp)
 	if (busy_oc != NULL) {
 		/* There are one or more busy objects, wait for them */
 		if (req->esi_level == 0) {
-			CHECK_OBJ_NOTNULL(sp->wrk->nwaitinglist,
+			CHECK_OBJ_NOTNULL(wrk->nwaitinglist,
 			    WAITINGLIST_MAGIC);
 			if (oh->waitinglist == NULL) {
-				oh->waitinglist = sp->wrk->nwaitinglist;
-				sp->wrk->nwaitinglist = NULL;
+				oh->waitinglist = wrk->nwaitinglist;
+				wrk->nwaitinglist = NULL;
 			}
 			VTAILQ_INSERT_TAIL(&oh->waitinglist->list, sp, list);
 		}
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 7ba9c60..a7bb31c 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -60,7 +60,7 @@
 #include "vend.h"
 
 struct vsb *
-VRY_Create(const struct sess *sp, const struct http *hp)
+VRY_Create(struct req *req, const struct http *hp)
 {
 	char *v, *p, *q, *h, *e;
 	struct vsb *sb, *sbh;
@@ -79,7 +79,7 @@ VRY_Create(const struct sess *sp, const struct http *hp)
 	AN(sbh);
 
 	if (*v == ':') {
-		VSLb(sp->req->vsl, SLT_Error,
+		VSLb(req->vsl, SLT_Error,
 		    "Vary header had extra ':', fix backend");
 		v++;
 	}
@@ -97,7 +97,7 @@ VRY_Create(const struct sess *sp, const struct http *hp)
 		    (char)(1 + (q - p)), (int)(q - p), p, 0);
 		AZ(VSB_finish(sbh));
 
-		if (http_GetHdr(sp->req->http, VSB_data(sbh), &h)) {
+		if (http_GetHdr(req->http, VSB_data(sbh), &h)) {
 			AZ(vct_issp(*h));
 			/* Trim trailing space */
 			e = strchr(h, '\0');
@@ -175,9 +175,9 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2)
 }
 
 int
-VRY_Match(const struct sess *sp, const uint8_t *vary)
+VRY_Match(struct req *req, const uint8_t *vary)
 {
-	uint8_t *vsp = sp->req->vary_b;
+	uint8_t *vsp = req->vary_b;
 	char *h, *e;
 	unsigned lh, ln;
 	int i, retval = 1, oflo = 0;
@@ -188,7 +188,7 @@ VRY_Match(const struct sess *sp, const uint8_t *vary)
 		if (i == 1) {
 			/* Build a new entry */
 
-			i = http_GetHdr(sp->req->http,
+			i = http_GetHdr(req->http,
 			    (const char*)(vary+2), &h);
 			if (i) {
 				/* Trim trailing space */
@@ -204,8 +204,8 @@ VRY_Match(const struct sess *sp, const uint8_t *vary)
 
 			/* Length of the entire new vary entry */
 			ln = 2 + vary[2] + 2 + (lh == 0xffff ? 0 : lh);
-			if (vsp + ln >= sp->req->vary_e) {
-				vsp = sp->req->vary_b;
+			if (vsp + ln >= req->vary_e) {
+				vsp = req->vary_b;
 				oflo = 1;
 			}
 
@@ -213,7 +213,7 @@ VRY_Match(const struct sess *sp, const uint8_t *vary)
 			 * We MUST have space for one entry and the end marker
 			 * after it, which prevents old junk from confusing us
 			 */
-			assert(vsp + ln + 2 < sp->req->vary_e);
+			assert(vsp + ln + 2 < req->vary_e);
 
 			vbe16enc(vsp, (uint16_t)lh);
 			memcpy(vsp + 2, vary + 2, vary[2] + 2);
@@ -231,20 +231,20 @@ VRY_Match(const struct sess *sp, const uint8_t *vary)
 		vsp += vry_len(vsp);
 		vary += vry_len(vary);
 	}
-	if (vsp + 3 > sp->req->vary_e)
+	if (vsp + 3 > req->vary_e)
 		oflo = 1;
 
 	if (oflo) {
 		/* XXX: Should log this */
-		vsp = sp->req->vary_b;
+		vsp = req->vary_b;
 	}
 	vsp[0] = 0xff;
 	vsp[1] = 0xff;
 	vsp[2] = 0;
 	if (oflo)
-		sp->req->vary_l = NULL;
+		req->vary_l = NULL;
 	else
-		sp->req->vary_l = vsp + 3;
+		req->vary_l = vsp + 3;
 	return (retval);
 }
 



From martin at varnish-cache.org  Mon Mar 12 15:50:06 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Mon, 12 Mar 2012 16:50:06 +0100
Subject: [master] c92ad59 Utilize the oc->priv2 variable to store the pointer
	to the allocating stevedore for the default case (malloc and file)
Message-ID: <E1S77Ve-0002gn-Ig@project.varnish-software.com>

commit c92ad59cde5dfd32487eb6466b07c4817e5efad1
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 12 16:11:01 2012 +0100

    Utilize the oc->priv2 variable to store the pointer to the allocating
    stevedore for the default case (malloc and file)
    
    Change the type of oc->priv2 from unsigned to uintptr_t (due to
    padding this does not increase the size of objcore).
    
    Change the default_oc_getlru to find the stevedore through the
    oc->priv2 instead of having to go through (and page-in) the object.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 6aceab0..8f2381c 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -371,7 +371,7 @@ struct objcore {
 	int			refcnt;
 	struct objcore_methods	*methods;
 	void			*priv;
-	unsigned		priv2;
+	uintptr_t		priv2;
 	struct objhead		*objhead;
 	struct busyobj		*busyobj;
 	double			timer_when;
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 3c62d8d..9f013d2 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -85,10 +85,10 @@ default_oc_freeobj(struct objcore *oc)
 static struct lru *
 default_oc_getlru(const struct objcore *oc)
 {
-	struct object *o;
+	struct stevedore *stv;
 
-	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
-	return (o->objstore->stevedore->lru);
+	CAST_OBJ_NOTNULL(stv, (void *)oc->priv2, STEVEDORE_MAGIC);
+	return (stv->lru);
 }
 
 static struct objcore_methods default_oc_methods = {
@@ -227,12 +227,13 @@ struct stv_objsecrets {
  */
 
 struct object *
-STV_MkObject(struct busyobj *bo, struct objcore **ocp, void *ptr, unsigned ltot,
-    const struct stv_objsecrets *soc)
+STV_MkObject(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
+    void *ptr, unsigned ltot, const struct stv_objsecrets *soc)
 {
 	struct object *o;
 	unsigned l;
 
+	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC);
 	AN(ocp);
@@ -270,6 +271,7 @@ STV_MkObject(struct busyobj *bo, struct objcore **ocp, void *ptr, unsigned ltot,
 
 		o->objcore->methods = &default_oc_methods;
 		o->objcore->priv = o;
+		o->objcore->priv2 = (uintptr_t)stv;
 	}
 	return (o);
 }
@@ -297,7 +299,7 @@ stv_default_allocobj(struct stevedore *stv, struct busyobj *bo,
 		return (NULL);
 	}
 	ltot = st->len = st->space;
-	o = STV_MkObject(bo, ocp, st->ptr, ltot, soc);
+	o = STV_MkObject(stv, bo, ocp, st->ptr, ltot, soc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	o->objstore = st;
 	return (o);
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index 80839a6..6994314 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -92,8 +92,9 @@ extern struct stevedore *stv_transient;
 int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx);
 uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,
     const char *ctx);
-struct object *STV_MkObject(struct busyobj *bo, struct objcore **ocp,
-    void *ptr, unsigned ltot, const struct stv_objsecrets *soc);
+struct object *STV_MkObject(struct stevedore *stv, struct busyobj *bo,
+    struct objcore **ocp, void *ptr, unsigned ltot,
+    const struct stv_objsecrets *soc);
 
 struct lru *LRU_Alloc(void);
 void LRU_Free(struct lru *lru);
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 58c6f29..35845f0 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -485,7 +485,7 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
 	assert(st->space >= ltot);
 	ltot = st->len = st->space;
 
-	o = STV_MkObject(bo, ocp, st->ptr, ltot, soc);
+	o = STV_MkObject(stv, bo, ocp, st->ptr, ltot, soc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	o->objstore = st;
 



From phk at varnish-cache.org  Mon Mar 12 19:35:14 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 20:35:14 +0100
Subject: [master] 88298f1 More sp -> req argument weakening
Message-ID: <E1S7B1W-0006ei-3e@project.varnish-software.com>

commit 88298f16110c19173d0b27de4a3d8417867a192f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 19:35:00 2012 +0000

    More sp -> req argument weakening

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8f2381c..17f4f4d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -749,8 +749,8 @@ void EXP_Set_ttl(struct exp *e, double v);
 void EXP_Set_grace(struct exp *e, double v);
 void EXP_Set_keep(struct exp *e, double v);
 
-double EXP_Ttl(const struct sess *, const struct object*);
-double EXP_Grace(const struct sess *, const struct object*);
+double EXP_Ttl(const struct req *, const struct object*);
+double EXP_Grace(const struct req *, const struct object*);
 void EXP_Insert(struct object *o);
 void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
 void EXP_Init(void);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 98b664f..9d7136c 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -109,39 +109,39 @@ EXP_ACCESS(keep, 0.,)
  */
 
 static double
-EXP_Keep(const struct sess *sp, const struct object *o)
+EXP_Keep(const struct req *req, const struct object *o)
 {
 	double r;
 
 	r = (double)cache_param->default_keep;
 	if (o->exp.keep > 0.)
 		r = o->exp.keep;
-	if (sp != NULL && sp->req->exp.keep > 0. && sp->req->exp.keep < r)
-		r = sp->req->exp.keep;
-	return (EXP_Ttl(sp, o) + r);
+	if (req != NULL && req->exp.keep > 0. && req->exp.keep < r)
+		r = req->exp.keep;
+	return (EXP_Ttl(req, o) + r);
 }
 
 double
-EXP_Grace(const struct sess *sp, const struct object *o)
+EXP_Grace(const struct req *req, const struct object *o)
 {
 	double r;
 
 	r = (double)cache_param->default_grace;
 	if (o->exp.grace >= 0.)
 		r = o->exp.grace;
-	if (sp != NULL && sp->req->exp.grace > 0. && sp->req->exp.grace < r)
-		r = sp->req->exp.grace;
-	return (EXP_Ttl(sp, o) + r);
+	if (req != NULL && req->exp.grace > 0. && req->exp.grace < r)
+		r = req->exp.grace;
+	return (EXP_Ttl(req, o) + r);
 }
 
 double
-EXP_Ttl(const struct sess *sp, const struct object *o)
+EXP_Ttl(const struct req *req, const struct object *o)
 {
 	double r;
 
 	r = o->exp.ttl;
-	if (sp != NULL && sp->req->exp.ttl > 0. && sp->req->exp.ttl < r)
-		r = sp->req->exp.ttl;
+	if (req != NULL && req->exp.ttl > 0. && req->exp.ttl < r)
+		r = req->exp.ttl;
 	return (o->exp.entered + r);
 }
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 34e86a4..6a3af2f 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -357,14 +357,14 @@ HSH_Lookup(struct sess *sp)
 			continue;
 
 		/* If still valid, use it */
-		if (EXP_Ttl(sp, o) >= sp->t_req)
+		if (EXP_Ttl(req, o) >= sp->t_req)
 			break;
 
 		/*
 		 * Remember any matching objects inside their grace period
 		 * and if there are several, use the least expired one.
 		 */
-		if (EXP_Grace(sp, o) >= sp->t_req) {
+		if (EXP_Grace(req, o) >= sp->t_req) {
 			if (grace_oc == NULL ||
 			    grace_ttl < o->exp.entered + o->exp.ttl) {
 				grace_oc = oc;



From phk at varnish-cache.org  Mon Mar 12 22:20:11 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 23:20:11 +0100
Subject: [master] cf81441 Tag oc's with COMPLETE or FAILURE when we fetch the
	body.
Message-ID: <E1S7Db9-0005Ml-0A@project.varnish-software.com>

commit cf81441e52a10143c5383472c9d25a073b6fde46
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 22:19:48 2012 +0000

    Tag oc's with COMPLETE or FAILURE when we fetch the body.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 17f4f4d..08e3405 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -381,6 +381,8 @@ struct objcore {
 #define OC_F_LRUDONTMOVE	(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
 #define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
+#define OC_F_COMPLETE		(1<<8)
+#define OC_F_FAILED		(1<<9)
 	unsigned		timer_idx;
 	VTAILQ_ENTRY(objcore)	list;
 	VTAILQ_ENTRY(objcore)	lru_list;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 1192f81..9f69bfa 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -921,9 +921,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	AN(req->director);
 
 	if (bo->state == BOS_FAILED) {
-		HSH_Drop(wrk, &sp->req->obj);
-		VBO_DerefBusyObj(wrk, &req->busyobj);
+		HSH_Drop(wrk, &req->obj);
 		AZ(req->obj);
+		VBO_DerefBusyObj(wrk, &req->busyobj);
 		req->err_code = 503;
 		sp->step = STP_ERROR;
 		return (0);
@@ -1145,6 +1145,9 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 		return (0);
 	}
 
+	/* For now... */
+	AN(oc->flags & (OC_F_COMPLETE|OC_F_FAILED));
+
 	o = oc_getobj(&wrk->stats, oc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	req->obj = o;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 172f422..feebd9b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -634,6 +634,13 @@ FetchBody(struct worker *wrk, void *priv)
 	http_Teardown(bo->bereq);
 	http_Teardown(bo->beresp);
 
+	if (obj->objcore != NULL) {
+		/* pass has no objcore */
+		/* XXX: lock protection ?? */
+		obj->objcore->flags |=
+		    (bo->state == BOS_FAILED ? OC_F_FAILED : OC_F_COMPLETE);
+	}
+
 	if (bo->state == BOS_FAILED) {
 		wrk->stats.fetch_failed++;
 		VDI_CloseFd(&bo->vbc);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 6a3af2f..514dbeb 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -269,6 +269,7 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc)
 	oc->refcnt = 1;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	AZ(oc->flags & OC_F_BUSY);
+	oc->flags |= OC_F_COMPLETE;
 
 	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
 	/* NB: do not deref objhead the new object inherits our reference */



From phk at varnish-cache.org  Mon Mar 12 22:45:28 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 12 Mar 2012 23:45:28 +0100
Subject: [master] 0041032 Let the body-fetching thread Unbusy the object.
Message-ID: <E1S7Dzc-00005r-Kx@project.varnish-software.com>

commit 0041032c2b01f865ed83adcf94e8789a3f31e2ce
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 22:45:04 2012 +0000

    Let the body-fetching thread Unbusy the object.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 9f69bfa..bc22656 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -929,13 +929,6 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 		return (0);
 	}
 
-	if (req->obj->objcore != NULL) {
-		EXP_Insert(req->obj);
-		AN(req->obj->objcore);
-		AN(req->obj->objcore->ban);
-		AZ(req->obj->ws_o->overflow);
-		HSH_Unbusy(&wrk->stats, req->obj->objcore);
-	}
 	VBO_DerefBusyObj(wrk, &req->busyobj);
 	wrk->acct_tmp.fetch++;
 	sp->step = STP_PREPRESP;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index feebd9b..968db87 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -35,6 +35,8 @@
 
 #include "cache.h"
 
+#include "hash/hash_slinger.h"
+
 #include "cache_backend.h"
 #include "vcli_priv.h"
 #include "vct.h"
@@ -674,13 +676,22 @@ FetchBody(struct worker *wrk, void *priv)
 			    "Content-Length: %zd", obj->len);
 		}
 
-		bo->state = BOS_FINISHED;
 
 		if (cls)
 			VDI_CloseFd(&bo->vbc);
 		else
 			VDI_RecycleFd(&bo->vbc);
 
+		if (obj->objcore != NULL) {
+			EXP_Insert(obj);
+			AN(obj->objcore->ban);
+			AZ(obj->ws_o->overflow);
+			HSH_Unbusy(&wrk->stats, obj->objcore);
+		}
+
+		/* XXX: Atomic assignment, needs volatile/membar ? */
+		bo->state = BOS_FINISHED;
+
 	}
 	bo->stats = NULL;
 	VBO_DerefBusyObj(wrk, &bo);



From phk at varnish-cache.org  Mon Mar 12 23:21:47 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 13 Mar 2012 00:21:47 +0100
Subject: [master] 2b21621 Also move dropping failed objects to FetchBody()
Message-ID: <E1S7EYl-0005CM-Dd@project.varnish-software.com>

commit 2b216210df2a797ff959dfdfe0231cf55c43db49
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 12 23:21:27 2012 +0000

    Also move dropping failed objects to FetchBody()

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index bc22656..4ad05ff 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -917,12 +917,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	assert(bo->state >= BOS_FAILED);
 
 	assert(WRW_IsReleased(wrk));
-	AZ(bo->vbc);
-	AN(req->director);
 
 	if (bo->state == BOS_FAILED) {
-		HSH_Drop(wrk, &req->obj);
-		AZ(req->obj);
+		req->obj = NULL;
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		req->err_code = 503;
 		sp->step = STP_ERROR;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 968db87..d9b519d 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -647,6 +647,7 @@ FetchBody(struct worker *wrk, void *priv)
 		wrk->stats.fetch_failed++;
 		VDI_CloseFd(&bo->vbc);
 		obj->len = 0;
+		HSH_Drop(wrk, &obj);
 	} else {
 		assert(bo->state == BOS_FETCHING);
 



From martin at varnish-cache.org  Tue Mar 13 14:50:52 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Tue, 13 Mar 2012 15:50:52 +0100
Subject: [master] d74bc62 Add a mutex timing flag to diag_bitmap
Message-ID: <E1S7T3s-0001tk-Jh@project.varnish-software.com>

commit d74bc62ea6173ed270b58c2da0c97c52323eec0b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 12 10:14:45 2012 +0100

    Add a mutex timing flag to diag_bitmap
    
    The MTX_UNLOCK debug lines will log for how long this thread
    held the lock.
    
    The MTX_LOCKWAIT debug lines will log how long the thread waited for the
    lock to be acquired.
    
    Bugs: Threads unlocking the mutex as part of pthread_cond_wait will
    not update the time of lock value for that mutex on pthread_cond_wait
    entry.

diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 9fb34ae..3bbbfbb 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -37,6 +37,7 @@
 
 #include <stdlib.h>
 
+#include "vtim.h"
 #include "cache.h"
 
 /*The constability of lck depends on platform pthreads implementation */
@@ -47,6 +48,7 @@ struct ilck {
 	pthread_mutex_t		mtx;
 	int			held;
 	pthread_t		owner;
+	double			t0;
 	VTAILQ_ENTRY(ilck)	list;
 	const char		*w;
 	struct VSC_C_lck	*stat;
@@ -62,9 +64,10 @@ Lck__Lock(struct lock *lck, const char *p, const char *f, int l)
 {
 	struct ilck *ilck;
 	int r;
+	double t0, t;
 
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	if (!(cache_param->diag_bitmap & 0x18)) {
+	if (!(cache_param->diag_bitmap & 0x98)) {
 		AZ(pthread_mutex_lock(&ilck->mtx));
 		AZ(ilck->held);
 		ilck->stat->locks++;
@@ -72,6 +75,8 @@ Lck__Lock(struct lock *lck, const char *p, const char *f, int l)
 		ilck->held = 1;
 		return;
 	}
+	if (cache_param->diag_bitmap & 0x80)
+		t0 = VTIM_real();
 	r = pthread_mutex_trylock(&ilck->mtx);
 	assert(r == 0 || r == EBUSY);
 	if (r) {
@@ -83,6 +88,12 @@ Lck__Lock(struct lock *lck, const char *p, const char *f, int l)
 	} else if (cache_param->diag_bitmap & 0x8) {
 		VSL(SLT_Debug, 0, "MTX_LOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
 	}
+	if (cache_param->diag_bitmap & 0x80) {
+		t = VTIM_real();
+		VSL(SLT_Debug, 0, "MTX_LOCKWAIT(%s,%s,%d,%s) %.9fs",
+		    p, f, l, ilck->w, t - t0);
+		ilck->t0 = t;
+	}
 	AZ(ilck->held);
 	ilck->stat->locks++;
 	ilck->owner = pthread_self();
@@ -110,7 +121,10 @@ Lck__Unlock(struct lock *lck, const char *p, const char *f, int l)
 	 */
 	memset(&ilck->owner, 0, sizeof ilck->owner);
 	AZ(pthread_mutex_unlock(&ilck->mtx));
-	if (cache_param->diag_bitmap & 0x8)
+	if (cache_param->diag_bitmap & 0x80)
+		VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s) %.9fs",
+		    p, f, l, ilck->w, VTIM_real() - ilck->t0);
+	else if (cache_param->diag_bitmap & 0x8)
 		VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
 }
 
@@ -131,6 +145,8 @@ Lck__Trylock(struct lock *lck, const char *p, const char *f, int l)
 		ilck->held = 1;
 		ilck->stat->locks++;
 		ilck->owner = pthread_self();
+		if (cache_param->diag_bitmap & 0x80)
+			ilck->t0 = VTIM_real();
 	}
 	return (r);
 }
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 31ef010..f6493ad 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1060,6 +1060,7 @@ static const struct parspec input_parspec[] = {
 		"  0x00000010 - mutex contests.\n"
 		"  0x00000020 - waiting list.\n"
 		"  0x00000040 - object workspace.\n"
+		"  0x00000080 - mutex timing.\n"
 		"  0x00001000 - do not core-dump child process.\n"
 		"  0x00002000 - only short panic message.\n"
 		"  0x00004000 - panic to stderr.\n"
@@ -1069,7 +1070,11 @@ static const struct parspec input_parspec[] = {
 		"  0x00080000 - ban-lurker debugging.\n"
 		"  0x80000000 - do edge-detection on digest.\n"
 		"\n"
-		"Use 0x notation and do the bitor in your head :-)\n",
+		"Use 0x notation and do the bitor in your head :-)\n"
+		"\n"
+		"Note: Mutex timing will add extra overhead and "
+		"synchronization between threads, consequently changing the "
+		"locking characteristics.\n",
 		0,
 		"0", "bitmap" },
 	{ "ban_dups", tweak_bool, &mgt_param.ban_dups, 0, 0,



From phk at varnish-cache.org  Wed Mar 14 18:35:13 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 14 Mar 2012 19:35:13 +0100
Subject: [master] 6393f26 Fix a misfeature in the hasher-API: Pass the digest
	in, so we don't have to touch the (potential) new objhead,
	unless we intend to insert it.
Message-ID: <E1S7t2X-0005Se-TX@project.varnish-software.com>

commit 6393f26c38d4c5bd7a6391ab1a016ed79b9830d7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 14 18:33:48 2012 +0000

    Fix a misfeature in the hasher-API:  Pass the digest in, so we don't have
    to touch the (potential) new objhead, unless we intend to insert it.
    
    Straighten out the colvolved logic of critbit while here.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 514dbeb..d154717 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -253,15 +253,10 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
 	hsh_prealloc(wrk);
-	if (cache_param->diag_bitmap & 0x80000000)
-		hsh_testmagic(wrk->nobjhead->digest);
 
 	AN(wrk->nobjhead);
-	memcpy(wrk->nobjhead->digest, digest, SHA256_LEN);
-	oh = hash->lookup(wrk, wrk->nobjhead);
+	oh = hash->lookup(wrk, digest, &wrk->nobjhead);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	if (oh == wrk->nobjhead)
-		wrk->nobjhead = NULL;
 	Lck_Lock(&oh->mtx);
 	assert(oh->refcnt > 0);
 
@@ -303,9 +298,8 @@ HSH_Lookup(struct sess *sp)
 	AN(hash);
 
 	hsh_prealloc(wrk);
-	memcpy(wrk->nobjhead->digest, req->digest, sizeof req->digest);
 	if (cache_param->diag_bitmap & 0x80000000)
-		hsh_testmagic(wrk->nobjhead->digest);
+		hsh_testmagic(req->digest);
 
 	if (req->hash_objhead != NULL) {
 		/*
@@ -317,9 +311,7 @@ HSH_Lookup(struct sess *sp)
 		req->hash_objhead = NULL;
 	} else {
 		AN(wrk->nobjhead);
-		oh = hash->lookup(wrk, wrk->nobjhead);
-		if (oh == wrk->nobjhead)
-			wrk->nobjhead = NULL;
+		oh = hash->lookup(wrk, req->digest, &wrk->nobjhead);
 	}
 
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c
index c84238a..87e0561 100644
--- a/bin/varnishd/hash/hash_classic.c
+++ b/bin/varnishd/hash/hash_classic.c
@@ -111,24 +111,26 @@ hcl_start(void)
  */
 
 static struct objhead * __match_proto__(hash_lookup_f)
-hcl_lookup(struct worker *wrk, struct objhead *noh)
+hcl_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 {
 	struct objhead *oh;
 	struct hcl_hd *hp;
-	unsigned u1, digest;
+	unsigned u1, hdigest;
 	int i;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
+	AN(digest);
+	AN(noh);
+	CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC);
 
-	assert(sizeof noh->digest > sizeof digest);
-	memcpy(&digest, noh->digest, sizeof digest);
-	u1 = digest % hcl_nhash;
+	assert(sizeof oh->digest >= sizeof hdigest);
+	memcpy(&hdigest, digest, sizeof hdigest);
+	u1 = hdigest % hcl_nhash;
 	hp = &hcl_head[u1];
 
 	Lck_Lock(&hp->mtx);
 	VTAILQ_FOREACH(oh, &hp->head, hoh_list) {
-		i = memcmp(oh->digest, noh->digest, sizeof oh->digest);
+		i = memcmp(oh->digest, digest, sizeof oh->digest);
 		if (i < 0)
 			continue;
 		if (i > 0)
@@ -139,14 +141,18 @@ hcl_lookup(struct worker *wrk, struct objhead *noh)
 	}
 
 	if (oh != NULL)
-		VTAILQ_INSERT_BEFORE(oh, noh, hoh_list);
+		VTAILQ_INSERT_BEFORE(oh, *noh, hoh_list);
 	else
-		VTAILQ_INSERT_TAIL(&hp->head, noh, hoh_list);
+		VTAILQ_INSERT_TAIL(&hp->head, *noh, hoh_list);
 
-	noh->hoh_head = hp;
+	oh = *noh;
+	*noh = NULL;
+	memcpy(oh->digest, digest, sizeof oh->digest);
+
+	oh->hoh_head = hp;
 
 	Lck_Unlock(&hp->mtx);
-	return (noh);
+	return (oh);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index 535661e..5bdab86 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -168,16 +168,15 @@ hcb_l_y(uintptr_t u)
  */
 
 static unsigned
-hcb_crit_bit(const struct objhead *oh1, const struct objhead *oh2,
-    struct hcb_y *y)
+hcb_crit_bit(const uint8_t *digest, const struct objhead *oh2, struct hcb_y *y)
 {
 	unsigned char u, r;
 
 	CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
-	for (u = 0; u < DIGEST_LEN && oh1->digest[u] == oh2->digest[u]; u++)
+	for (u = 0; u < DIGEST_LEN && digest[u] == oh2->digest[u]; u++)
 		;
 	assert(u < DIGEST_LEN);
-	r = hcb_bits(oh1->digest[u], oh2->digest[u]);
+	r = hcb_bits(digest[u], oh2->digest[u]);
 	y->ptr = u;
 	y->bitmask = 0x80 >> r;
 	y->critbit = u * 8 + r;
@@ -191,8 +190,8 @@ hcb_crit_bit(const struct objhead *oh1, const struct objhead *oh2,
  */
 
 static struct objhead *
-hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh,
-    int has_lock)
+hcb_insert(struct worker *wrk, struct hcb_root *root, const uint8_t *digest,
+    struct objhead **noh)
 {
 	volatile uintptr_t *p;
 	uintptr_t pp;
@@ -203,17 +202,20 @@ hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh,
 	p = &root->origo;
 	pp = *p;
 	if (pp == 0) {
-		if (!has_lock)
+		if (noh == NULL)
 			return (NULL);
-		*p = hcb_r_node(oh);
-		return (oh);
+		oh2 = *noh;
+		*noh = NULL;
+		memcpy(oh2->digest, digest, sizeof oh2->digest);
+		*p = hcb_r_node(oh2);
+		return (oh2);
 	}
 
 	while(hcb_is_y(pp)) {
 		y = hcb_l_y(pp);
 		CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
 		assert(y->ptr < DIGEST_LEN);
-		s = (oh->digest[y->ptr] & y->bitmask) != 0;
+		s = (digest[y->ptr] & y->bitmask) != 0;
 		assert(s < 2);
 		p = &y->leaf[s];
 		pp = *p;
@@ -221,7 +223,7 @@ hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh,
 
 	if (pp == 0) {
 		/* We raced hcb_delete and got a NULL pointer */
-		assert(!has_lock);
+		assert(noh == NULL);
 		return (NULL);
 	}
 
@@ -230,20 +232,23 @@ hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh,
 	/* We found a node, does it match ? */
 	oh2 = hcb_l_node(pp);
 	CHECK_OBJ_NOTNULL(oh2, OBJHEAD_MAGIC);
-	if (!memcmp(oh2->digest, oh->digest, DIGEST_LEN))
+	if (!memcmp(oh2->digest, digest, DIGEST_LEN))
 		return (oh2);
 
-	if (!has_lock)
+	if (noh == NULL)
 		return (NULL);
 
 	/* Insert */
 
 	CAST_OBJ_NOTNULL(y2, wrk->nhashpriv, HCB_Y_MAGIC);
 	wrk->nhashpriv = NULL;
-	(void)hcb_crit_bit(oh, oh2, y2);
-	s2 = (oh->digest[y2->ptr] & y2->bitmask) != 0;
+	(void)hcb_crit_bit(digest, oh2, y2);
+	s2 = (digest[y2->ptr] & y2->bitmask) != 0;
 	assert(s2 < 2);
-	y2->leaf[s2] = hcb_r_node(oh);
+	oh2 = *noh;
+	*noh = NULL;
+	memcpy(oh2->digest, digest, sizeof oh2->digest);
+	y2->leaf[s2] = hcb_r_node(oh2);
 	s2 = 1-s2;
 
 	p = &root->origo;
@@ -256,14 +261,14 @@ hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh,
 		if (y->critbit > y2->critbit)
 			break;
 		assert(y->ptr < DIGEST_LEN);
-		s = (oh->digest[y->ptr] & y->bitmask) != 0;
+		s = (digest[y->ptr] & y->bitmask) != 0;
 		assert(s < 2);
 		p = &y->leaf[s];
 	}
 	y2->leaf[s2] = *p;
 	VWMB();
 	*p = hcb_r_y(y2);
-	return(oh);
+	return(oh2);
 }
 
 /*--------------------------------------------------------------------*/
@@ -412,45 +417,50 @@ hcb_deref(struct objhead *oh)
 }
 
 static struct objhead * __match_proto__(hash_lookup_f)
-hcb_lookup(struct worker *wrk, struct objhead *noh)
+hcb_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 {
 	struct objhead *oh;
 	struct hcb_y *y;
 	unsigned u;
-	unsigned with_lock;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AN(digest);
+	AN(noh);
+	CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC);
+	assert((*noh)->refcnt == 1);
+
+	/* First try in read-only mode without holding a lock */
+
+	VSC_C_main->hcb_nolock++;
+	oh = hcb_insert(wrk, &hcb_root, digest, NULL);
+	if (oh != NULL) {
+		Lck_Lock(&oh->mtx);
+		/*
+		 * A refcount of zero indicates that the tree changed
+		 * under us, so fall through and try with the lock held.
+		 */
+		u = oh->refcnt;
+		if (u > 0)
+			oh->refcnt++;
+		Lck_Unlock(&oh->mtx);
+		if (u > 0)
+			return (oh);
+	}
 
-	with_lock = 0;
 	while (1) {
-		if (with_lock) {
-			CAST_OBJ_NOTNULL(y, wrk->nhashpriv, HCB_Y_MAGIC);
-			Lck_Lock(&hcb_mtx);
-			VSC_C_main->hcb_lock++;
-			assert(noh->refcnt == 1);
-			oh = hcb_insert(wrk, &hcb_root, noh, 1);
-			Lck_Unlock(&hcb_mtx);
-		} else {
-			VSC_C_main->hcb_nolock++;
-			oh = hcb_insert(wrk, &hcb_root, noh, 0);
-		}
+		/* No luck, try with lock held, so we can modify tree */
+		CAST_OBJ_NOTNULL(y, wrk->nhashpriv, HCB_Y_MAGIC);
+		Lck_Lock(&hcb_mtx);
+		VSC_C_main->hcb_lock++;
+		oh = hcb_insert(wrk, &hcb_root, digest, noh);
+		Lck_Unlock(&hcb_mtx);
 
-		if (oh != NULL && oh == noh) {
-			/* Assert that we only muck with the tree with a lock */
-			assert(with_lock);
-			VSC_C_main->hcb_insert++;
+		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+		if (*noh == NULL) {
 			assert(oh->refcnt > 0);
+			VSC_C_main->hcb_insert++;
 			return (oh);
 		}
-
-		if (oh == NULL) {
-			assert(!with_lock);
-			/* Try again, with lock */
-			with_lock = 1;
-			continue;
-		}
-
-		CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
 		Lck_Lock(&oh->mtx);
 		/*
 		 * A refcount of zero indicates that the tree changed
@@ -459,8 +469,6 @@ hcb_lookup(struct worker *wrk, struct objhead *noh)
 		u = oh->refcnt;
 		if (u > 0)
 			oh->refcnt++;
-		else
-			with_lock = 1;
 		Lck_Unlock(&oh->mtx);
 		if (u > 0)
 			return (oh);
diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c
index fef2b65..d70e76f 100644
--- a/bin/varnishd/hash/hash_simple_list.c
+++ b/bin/varnishd/hash/hash_simple_list.c
@@ -60,16 +60,19 @@ hsl_start(void)
  */
 
 static struct objhead * __match_proto__(hash_lookup_f)
-hsl_lookup(struct worker *wrk, struct objhead *noh)
+hsl_lookup(struct worker *wrk, const void *digest, struct objhead **noh)
 {
 	struct objhead *oh;
 	int i;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
+	AN(digest);
+	AN(noh);
+	CHECK_OBJ_NOTNULL(*noh, OBJHEAD_MAGIC);
+
 	Lck_Lock(&hsl_mtx);
 	VTAILQ_FOREACH(oh, &hsl_head, hoh_list) {
-		i = memcmp(oh->digest, noh->digest, sizeof oh->digest);
+		i = memcmp(oh->digest, digest, sizeof oh->digest);
 		if (i < 0)
 			continue;
 		if (i > 0)
@@ -80,12 +83,15 @@ hsl_lookup(struct worker *wrk, struct objhead *noh)
 	}
 
 	if (oh != NULL)
-		VTAILQ_INSERT_BEFORE(oh, noh, hoh_list);
+		VTAILQ_INSERT_BEFORE(oh, *noh, hoh_list);
 	else
-		VTAILQ_INSERT_TAIL(&hsl_head, noh, hoh_list);
+		VTAILQ_INSERT_TAIL(&hsl_head, *noh, hoh_list);
 
+	oh = *noh;
+	*noh = NULL;
+	memcpy(oh->digest, digest, sizeof oh->digest);
 	Lck_Unlock(&hsl_mtx);
-	return (noh);
+	return (oh);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 4935bfb..628d080 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -35,8 +35,8 @@ struct object;
 typedef void hash_init_f(int ac, char * const *av);
 typedef void hash_start_f(void);
 typedef void hash_prep_f(struct worker *);
-typedef struct objhead *
-    hash_lookup_f(struct worker *wrk, struct objhead *nobj);
+typedef struct objhead *hash_lookup_f(struct worker *wrk, const void *digest,
+    struct objhead **nobj);
 typedef int hash_deref_f(struct objhead *obj);
 
 struct hash_slinger {



From phk at varnish-cache.org  Wed Mar 14 21:25:50 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 14 Mar 2012 22:25:50 +0100
Subject: [master] 4bbbc69 In theory diag_bitmap could change between
	(non-)assignment to t0 and use of t0. Always initialize t0 to
	something.
Message-ID: <E1S7vhe-0002iU-Tm@project.varnish-software.com>

commit 4bbbc69324104deaac64d3845237f0c95ec625f4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 14 21:24:42 2012 +0000

    In theory diag_bitmap could change between (non-)assignment to t0
    and use of t0.  Always initialize t0 to something.
    
    Correctly spotted by:	FlexeLint

diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 3bbbfbb..993bb5e 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -64,7 +64,7 @@ Lck__Lock(struct lock *lck, const char *p, const char *f, int l)
 {
 	struct ilck *ilck;
 	int r;
-	double t0, t;
+	double t0 = 0, t;
 
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
 	if (!(cache_param->diag_bitmap & 0x98)) {



From phk at varnish-cache.org  Wed Mar 14 21:25:50 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 14 Mar 2012 22:25:50 +0100
Subject: [master] dbad89e Now that saint-mode uses the digest,
	we can remove the temporary objcore assignment.
Message-ID: <E1S7vhe-0002iZ-Uv@project.varnish-software.com>

commit dbad89eb134506e5c4f8c521d40d685b8388579b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 14 21:25:25 2012 +0000

    Now that saint-mode uses the digest, we can remove the temporary
    objcore assignment.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index dcb87dd..92e74da 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -271,9 +271,6 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	if (threshold == 0 || VTAILQ_EMPTY(&backend->troublelist))
 		return (1);
 
-	if (sp->req->objcore == NULL)
-		return (1);
-
 	now = sp->t_req;
 
 	old = NULL;
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index d154717..749a8f3 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -377,7 +377,6 @@ HSH_Lookup(struct sess *sp)
 	 */
 
 	AZ(req->objcore);
-	req->objcore = grace_oc;		/* XXX: Hack-ish */
 	if (oc == NULL			/* We found no live object */
 	    && grace_oc != NULL		/* There is a grace candidate */
 	    && (busy_oc != NULL		/* Somebody else is already busy */
@@ -387,7 +386,6 @@ HSH_Lookup(struct sess *sp)
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		oc = grace_oc;
 	}
-	req->objcore = NULL;
 
 	if (oc != NULL && !req->hash_always_miss) {
 		o = oc_getobj(&wrk->stats, oc);



From phk at varnish-cache.org  Wed Mar 14 23:21:40 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 15 Mar 2012 00:21:40 +0100
Subject: [master] ca3099c Lessen the load on oh->mtx a bit.
Message-ID: <E1S7xVk-0001FQ-MO@project.varnish-software.com>

commit ca3099c3b6c0e52ebf37054163300a3bdaca554d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Mar 14 23:21:28 2012 +0000

    Lessen the load on oh->mtx a bit.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 08e3405..daab7da 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -383,6 +383,7 @@ struct objcore {
 #define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
 #define OC_F_COMPLETE		(1<<8)
 #define OC_F_FAILED		(1<<9)
+#define OC_F_NOTYET		(1<<10)
 	unsigned		timer_idx;
 	VTAILQ_ENTRY(objcore)	list;
 	VTAILQ_ENTRY(objcore)	lru_list;
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 749a8f3..06d7e50 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -326,6 +326,9 @@ HSH_Lookup(struct sess *sp)
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 		assert(oc->objhead == oh);
 
+		if (oc->flags & OC_F_NOTYET)
+			continue;
+
 		if (oc->flags & OC_F_BUSY) {
 			CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
 			if (req->hash_ignore_busy)
@@ -370,10 +373,7 @@ HSH_Lookup(struct sess *sp)
 	 * If we have seen a busy object or the backend is unhealthy, and
 	 * we have an object in grace, use it, if req.grace is also
 	 * satisified.
-	 * XXX: Interesting footnote:  The busy object might be for a
-	 * XXX: different "Vary:" than we sought.  We have no way of knowing
-	 * XXX: this until the object is unbusy'ed, so in practice we
-	 * XXX: serialize fetch of all Vary's if grace is possible.
+	 * XXX: VDI_Healty() call with oh->mtx is not so cool.
 	 */
 
 	AZ(req->objcore);
@@ -388,17 +388,16 @@ HSH_Lookup(struct sess *sp)
 	}
 
 	if (oc != NULL && !req->hash_always_miss) {
-		o = oc_getobj(&wrk->stats, oc);
-		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		assert(oc->objhead == oh);
-
 		/* We found an object we like */
-		oc->refcnt++;
-		if (o->hits < INT_MAX)
-			o->hits++;
 		assert(oh->refcnt > 1);
+		assert(oc->objhead == oh);
+		oc->refcnt++;
 		Lck_Unlock(&oh->mtx);
 		assert(hash->deref(oh));
+		o = oc_getobj(&wrk->stats, oc);
+		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+		if (o->hits < INT_MAX)
+			o->hits++;
 		return (oc);
 	}
 
@@ -428,11 +427,16 @@ HSH_Lookup(struct sess *sp)
 		return (NULL);
 	}
 
-	/* Insert (precreated) objcore in objecthead */
+	/* Insert (precreated) objcore in objecthead and release mutex */
 	oc = wrk->nobjcore;
 	wrk->nobjcore = NULL;
 	AN(oc->flags & OC_F_BUSY);
+	oc->flags |= OC_F_NOTYET;
 	oc->refcnt = 1;
+	oc->objhead = oh;
+	VTAILQ_INSERT_TAIL(&oh->objcs, oc, list);
+	/* NB: do not deref objhead the new object inherits our reference */
+	Lck_Unlock(&oh->mtx);
 
 	AZ(req->busyobj);
 	req->busyobj = VBO_GetBusyObj(wrk);
@@ -445,14 +449,7 @@ HSH_Lookup(struct sess *sp)
 		req->busyobj->vary = NULL;
 	oc->busyobj = req->busyobj;
 
-	/*
-	 * Busy objects go on the tail, so they will not trip up searches.
-	 * HSH_Unbusy() will move them to the front.
-	 */
-	VTAILQ_INSERT_TAIL(&oh->objcs, oc, list);
-	oc->objhead = oh;
-	/* NB: do not deref objhead the new object inherits our reference */
-	Lck_Unlock(&oh->mtx);
+	oc->flags &= ~OC_F_NOTYET;
 	return (oc);
 }
 



From phk at varnish-cache.org  Thu Mar 15 13:00:52 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 15 Mar 2012 14:00:52 +0100
Subject: [master] bbc0bc4 Start preparing hash-lookup for streaming objects
Message-ID: <E1S8AIW-0006Kv-2S@project.varnish-software.com>

commit bbc0bc445732fd451286f5bcce172fe7ffd4aa3c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 15 08:11:20 2012 +0000

    Start preparing hash-lookup for streaming objects

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 4ad05ff..c9498b6 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -654,6 +654,8 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 		AZ(HSH_Deref(&wrk->stats, req->objcore, NULL));
 		req->objcore = NULL;
 	}
+	assert(bo->refcount == 2);
+	VBO_DerefBusyObj(wrk, &bo);
 	VBO_DerefBusyObj(wrk, &req->busyobj);
 	req->director = NULL;
 	req->storage_hint = NULL;
@@ -906,8 +908,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	bo->fetch_task.func = FetchBody;
 	bo->fetch_task.priv = bo;
 
-	/* Gain a reference for FetchBody() */
-	VBO_RefBusyObj(bo);
+	assert(bo->refcount == 2);	/* one for each thread */
 
 	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
 		FetchBody(wrk, bo);
@@ -1264,6 +1265,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
 
 	req->busyobj = VBO_GetBusyObj(wrk);
 	req->busyobj->vsl->wid = sp->vsl_id;
+	req->busyobj->refcount = 2;
 	http_Setup(req->busyobj->bereq, req->busyobj->ws, req->busyobj->vsl);
 	http_FilterReq(sp, HTTPH_R_PASS);
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 06d7e50..60bf207 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -58,7 +58,9 @@
 
 #include "cache.h"
 
+
 #include "hash/hash_slinger.h"
+#include "vmb.h"
 #include "vsha256.h"
 
 static const struct hash_slinger *hash;
@@ -326,15 +328,17 @@ HSH_Lookup(struct sess *sp)
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 		assert(oc->objhead == oh);
 
-		if (oc->flags & OC_F_NOTYET)
+		/* We ignore failed oc's, they're no use, ever. */
+		if (oc->flags & OC_F_FAILED)
 			continue;
 
-		if (oc->flags & OC_F_BUSY) {
+		if (oc->flags & (OC_F_BUSY | OC_F_NOTYET)) {
 			CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
 			if (req->hash_ignore_busy)
 				continue;
 
-			if (oc->busyobj->vary != NULL &&
+			if (!(oc->flags & OC_F_NOTYET) &&
+			    oc->busyobj->vary != NULL &&
 			    !VRY_Match(req, oc->busyobj->vary))
 				continue;
 
@@ -440,15 +444,17 @@ HSH_Lookup(struct sess *sp)
 
 	AZ(req->busyobj);
 	req->busyobj = VBO_GetBusyObj(wrk);
+	oc->busyobj = req->busyobj;
 	req->busyobj->vsl->wid = sp->vsl_id;
+	req->busyobj->refcount = 2;	/* One for headers, one for body*/
 
 	VRY_Validate(req->vary_b);
 	if (req->vary_l != NULL)
 		req->busyobj->vary = req->vary_b;
 	else
 		req->busyobj->vary = NULL;
-	oc->busyobj = req->busyobj;
 
+	VMB();
 	oc->flags &= ~OC_F_NOTYET;
 	return (oc);
 }



From phk at varnish-cache.org  Mon Mar 19 09:28:36 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Mar 2012 10:28:36 +0100
Subject: [master] d734511 Update comment to match reality
Message-ID: <E1S9YtI-000395-Ph@project.varnish-software.com>

commit d7345115a31ddf7e80ded4b2b914ae17c0d2496a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 19 09:25:55 2012 +0000

    Update comment to match reality

diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index a7bb31c..138efda 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -49,7 +49,9 @@
  *	'\0'			/
  *      <header>		>   Only present if length != 0xffff
  * }
- *      '\0'
+ *	0xff,			\   Length field
+ *	0xff,			/
+ *      '\0'			>   Terminator
  */
 
 #include "config.h"



From phk at varnish-cache.org  Mon Mar 19 09:28:36 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Mar 2012 10:28:36 +0100
Subject: [master] 20d561d Redo the oc->NOTYET stuff after pondering.
Message-ID: <E1S9YtI-00039A-Qz@project.varnish-software.com>

commit 20d561d980c291aed95c61d58b87183721760c4d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 19 09:26:16 2012 +0000

    Redo the oc->NOTYET stuff after pondering.
    
    OC's will only have two states: BUSY or not BUSY.
    
    A BUSY OC may not yet have a busyobj, in which case we assume that
    it vary-matches.
    
    Replace the busy_oc pointer with flag, all we are about is the existence
    of a busy OC, we're not going to actually touch it.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index daab7da..17f4f4d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -381,9 +381,6 @@ struct objcore {
 #define OC_F_LRUDONTMOVE	(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
 #define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
-#define OC_F_COMPLETE		(1<<8)
-#define OC_F_FAILED		(1<<9)
-#define OC_F_NOTYET		(1<<10)
 	unsigned		timer_idx;
 	VTAILQ_ENTRY(objcore)	list;
 	VTAILQ_ENTRY(objcore)	lru_list;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index c9498b6..30a797b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1136,9 +1136,6 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 		return (0);
 	}
 
-	/* For now... */
-	AN(oc->flags & (OC_F_COMPLETE|OC_F_FAILED));
-
 	o = oc_getobj(&wrk->stats, oc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	req->obj = o;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index d9b519d..5406b2f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -636,13 +636,6 @@ FetchBody(struct worker *wrk, void *priv)
 	http_Teardown(bo->bereq);
 	http_Teardown(bo->beresp);
 
-	if (obj->objcore != NULL) {
-		/* pass has no objcore */
-		/* XXX: lock protection ?? */
-		obj->objcore->flags |=
-		    (bo->state == BOS_FAILED ? OC_F_FAILED : OC_F_COMPLETE);
-	}
-
 	if (bo->state == BOS_FAILED) {
 		wrk->stats.fetch_failed++;
 		VDI_CloseFd(&bo->vbc);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 60bf207..978277a 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -266,7 +266,6 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc)
 	oc->refcnt = 1;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	AZ(oc->flags & OC_F_BUSY);
-	oc->flags |= OC_F_COMPLETE;
 
 	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
 	/* NB: do not deref objhead the new object inherits our reference */
@@ -285,10 +284,11 @@ HSH_Lookup(struct sess *sp)
 	struct worker *wrk;
 	struct objhead *oh;
 	struct objcore *oc;
-	struct objcore *busy_oc, *grace_oc;
+	struct objcore *grace_oc;
 	struct object *o;
 	struct req *req;
 	double grace_ttl;
+	int busy_found;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
@@ -319,7 +319,7 @@ HSH_Lookup(struct sess *sp)
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 	Lck_Lock(&oh->mtx);
 	assert(oh->refcnt > 0);
-	busy_oc = NULL;
+	busy_found = 0;
 	grace_oc = NULL;
 	grace_ttl = NAN;
 	VTAILQ_FOREACH(oc, &oh->objcs, list) {
@@ -328,21 +328,17 @@ HSH_Lookup(struct sess *sp)
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 		assert(oc->objhead == oh);
 
-		/* We ignore failed oc's, they're no use, ever. */
-		if (oc->flags & OC_F_FAILED)
-			continue;
-
-		if (oc->flags & (OC_F_BUSY | OC_F_NOTYET)) {
-			CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
+		if (oc->flags & OC_F_BUSY) {
+			CHECK_OBJ_ORNULL(oc->busyobj, BUSYOBJ_MAGIC);
 			if (req->hash_ignore_busy)
 				continue;
 
-			if (!(oc->flags & OC_F_NOTYET) &&
+			if (oc->busyobj != NULL &&
 			    oc->busyobj->vary != NULL &&
 			    !VRY_Match(req, oc->busyobj->vary))
 				continue;
 
-			busy_oc = oc;
+			busy_found = 1;
 			continue;
 		}
 
@@ -383,7 +379,7 @@ HSH_Lookup(struct sess *sp)
 	AZ(req->objcore);
 	if (oc == NULL			/* We found no live object */
 	    && grace_oc != NULL		/* There is a grace candidate */
-	    && (busy_oc != NULL		/* Somebody else is already busy */
+	    && (busy_found 		/* Somebody else is already busy */
 	    || !VDI_Healthy(req->director, sp))) {
 					/* Or it is impossible to fetch */
 		o = oc_getobj(&wrk->stats, grace_oc);
@@ -405,7 +401,7 @@ HSH_Lookup(struct sess *sp)
 		return (oc);
 	}
 
-	if (busy_oc != NULL) {
+	if (busy_found) {
 		/* There are one or more busy objects, wait for them */
 		if (req->esi_level == 0) {
 			CHECK_OBJ_NOTNULL(wrk->nwaitinglist,
@@ -435,7 +431,6 @@ HSH_Lookup(struct sess *sp)
 	oc = wrk->nobjcore;
 	wrk->nobjcore = NULL;
 	AN(oc->flags & OC_F_BUSY);
-	oc->flags |= OC_F_NOTYET;
 	oc->refcnt = 1;
 	oc->objhead = oh;
 	VTAILQ_INSERT_TAIL(&oh->objcs, oc, list);
@@ -444,7 +439,6 @@ HSH_Lookup(struct sess *sp)
 
 	AZ(req->busyobj);
 	req->busyobj = VBO_GetBusyObj(wrk);
-	oc->busyobj = req->busyobj;
 	req->busyobj->vsl->wid = sp->vsl_id;
 	req->busyobj->refcount = 2;	/* One for headers, one for body*/
 
@@ -455,7 +449,7 @@ HSH_Lookup(struct sess *sp)
 		req->busyobj->vary = NULL;
 
 	VMB();
-	oc->flags &= ~OC_F_NOTYET;
+	oc->busyobj = req->busyobj;
 	return (oc);
 }
 



From phk at varnish-cache.org  Mon Mar 19 10:14:55 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Mar 2012 11:14:55 +0100
Subject: [master] bd0246a Edge closer to the new worldorder
Message-ID: <E1S9Zc7-0007UY-0k@project.varnish-software.com>

commit bd0246a0de438739a07bb1dca7f7badb67bc3bb5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 19 10:14:41 2012 +0000

    Edge closer to the new worldorder

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 17f4f4d..f337f66 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1081,21 +1081,6 @@ Tadd(txt *t, const char *p, int l)
 	}
 }
 
-static inline void
-AssertOCBusy(const struct objcore *oc)
-{
-	AN(oc);
-	AN (oc->flags & OC_F_BUSY);
-	AN(oc->busyobj);
-}
-
-static inline void
-AssertObjCorePassOrBusy(const struct objcore *oc)
-{
-	if (oc != NULL)
-		AN (oc->flags & OC_F_BUSY);
-}
-
 /*
  * We want to cache the most recent timestamp in wrk->lastused to avoid
  * extra timestamps in cache_pool.c.  Hide this detail with a macro
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 30a797b..8b12807 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -215,7 +215,6 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 	if (bo != NULL) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 		AN(bo->do_stream);
-		AssertObjCorePassOrBusy(req->obj->objcore);
 	}
 
 	req->res_mode = 0;
@@ -635,7 +634,6 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 
 		switch (req->handling) {
 		case VCL_RET_DELIVER:
-			AssertObjCorePassOrBusy(req->objcore);
 			sp->step = STP_PREPFETCH;
 			return (0);
 		default:
@@ -875,8 +873,6 @@ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req)
 	    RFC2616_Do_Cond(sp))
 		bo->do_stream = 0;
 
-	AssertObjCorePassOrBusy(req->obj->objcore);
-
 	sp->step = STP_FETCHBODY;
 	return (0);
 }
@@ -1136,6 +1132,9 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 		return (0);
 	}
 
+	/* We are not prepared to do streaming yet */
+	XXXAZ(req->busyobj);
+
 	o = oc_getobj(&wrk->stats, oc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	req->obj = o;
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 9d7136c..39ded77 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -224,7 +224,6 @@ EXP_Insert(struct object *o)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AssertOCBusy(oc);
 	HSH_Ref(oc);
 
 	assert(o->exp.entered != 0 && !isnan(o->exp.entered));
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 5406b2f..cab6273 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -40,6 +40,7 @@
 #include "cache_backend.h"
 #include "vcli_priv.h"
 #include "vct.h"
+#include "vmb.h"
 #include "vtcp.h"
 
 static unsigned fetchfrag;
@@ -570,8 +571,6 @@ FetchBody(struct worker *wrk, void *priv)
 	if (bo->vfp == NULL)
 		bo->vfp = &vfp_nop;
 
-	AssertObjCorePassOrBusy(obj->objcore);
-
 	AZ(bo->vgz_rx);
 	AZ(VTAILQ_FIRST(&obj->store));
 
@@ -681,6 +680,8 @@ FetchBody(struct worker *wrk, void *priv)
 			AN(obj->objcore->ban);
 			AZ(obj->ws_o->overflow);
 			HSH_Unbusy(&wrk->stats, obj->objcore);
+			obj->objcore->busyobj = NULL;
+			VMB();
 		}
 
 		/* XXX: Atomic assignment, needs volatile/membar ? */
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 978277a..09ee9f4 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -551,9 +551,7 @@ HSH_Purge(const struct sess *sp, struct objhead *oh, double ttl, double grace)
 
 
 /*---------------------------------------------------------------------
- * Kill a busy object we don't need anyway.
- * There may be sessions on the waiting list, so we cannot just blow
- * it out of the water.
+ * Kill a busy object we don't need and can't use.
  */
 
 void
@@ -563,13 +561,14 @@ HSH_Drop(struct worker *wrk, struct object **oo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AN(oo);
 	CHECK_OBJ_NOTNULL(*oo, OBJECT_MAGIC);
-	AssertObjCorePassOrBusy((*oo)->objcore);
 	(*oo)->exp.ttl = -1.;
-	if ((*oo)->objcore != NULL)		/* Pass has no objcore */
-		HSH_Unbusy(&wrk->stats, (*oo)->objcore);
-	(void)HSH_Deref(&wrk->stats, NULL, oo);
+	AZ(HSH_Deref(&wrk->stats, NULL, oo));
 }
 
+/*---------------------------------------------------------------------
+ * Unbusy an objcore when the object is completely fetched.
+ */
+
 void
 HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 {
@@ -579,9 +578,8 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 	oh = oc->objhead;
 	CHECK_OBJ(oh, OBJHEAD_MAGIC);
 
-	AssertOCBusy(oc);
+	AN(oc->flags & OC_F_BUSY);
 	AN(oc->ban);
-	assert(oc->refcnt > 0);
 	assert(oh->refcnt > 0);
 
 	/* XXX: pretouch neighbors on oh->objcs to prevent page-on under mtx */
@@ -591,13 +589,15 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 	VTAILQ_REMOVE(&oh->objcs, oc, list);
 	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
 	oc->flags &= ~OC_F_BUSY;
-	oc->busyobj = NULL;
 	if (oh->waitinglist != NULL)
 		hsh_rush(ds, oh);
-	AN(oc->ban);
 	Lck_Unlock(&oh->mtx);
 }
 
+/*---------------------------------------------------------------------
+ * Gain a reference on an objcore
+ */
+
 void
 HSH_Ref(struct objcore *oc)
 {



From phk at varnish-cache.org  Mon Mar 19 13:38:10 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 19 Mar 2012 14:38:10 +0100
Subject: [master] c40a7d0 Unbusy the objectcore when we start fetching the
	body.
Message-ID: <E1S9cmo-0001ib-8n@project.varnish-software.com>

commit c40a7d0ce9ed192e1f1d482172dbe1f1c1dfe7db
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Mar 19 13:36:36 2012 +0000

    Unbusy the objectcore when we start fetching the body.
    
    Don't deref the objcore until the busyobj dies.
    
    Apply two workarounds for now:  Don't return non-busy object with
    a busyobj in HSH_Lookup(), and trade the busyobj ref for an objcore ref
    in cnt_fetchbody()
    
    Getting closer...

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 8de928a..79b55ed 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -38,6 +38,8 @@
 
 #include "cache.h"
 
+#include "hash/hash_slinger.h"
+
 static struct mempool		*vbopool;
 
 /*--------------------------------------------------------------------
@@ -157,6 +159,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 	bo = *pbo;
 	*pbo = NULL;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CHECK_OBJ_ORNULL(bo->fetch_obj, OBJECT_MAGIC);
 	Lck_Lock(&bo->mtx);
 	assert(bo->refcount > 0);
 	r = --bo->refcount;
@@ -167,6 +170,11 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 
 	VSL_Flush(bo->vsl, 0);
 
+	if (bo->fetch_obj != NULL && bo->fetch_obj->objcore != NULL) {
+		AN(wrk);
+		(void)HSH_Deref(&wrk->stats, NULL, &bo->fetch_obj);
+	}
+
 	memset(&bo->refcount, 0,
 	    sizeof *bo - offsetof(struct busyobj, refcount));
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 8b12807..a5b058b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -906,6 +906,13 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 
 	assert(bo->refcount == 2);	/* one for each thread */
 
+	if (req->obj->objcore != NULL) {
+		EXP_Insert(req->obj);
+		AN(req->obj->objcore->ban);
+		AZ(req->obj->ws_o->overflow);
+		HSH_Unbusy(&wrk->stats, req->obj->objcore);
+	}
+
 	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_NO_QUEUE))
 		FetchBody(wrk, bo);
 
@@ -923,6 +930,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 		return (0);
 	}
 
+	if (req->obj->objcore != NULL)
+		HSH_Ref(req->obj->objcore);
+
 	VBO_DerefBusyObj(wrk, &req->busyobj);
 	wrk->acct_tmp.fetch++;
 	sp->step = STP_PREPRESP;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index cab6273..d71a6cb 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -35,8 +35,6 @@
 
 #include "cache.h"
 
-#include "hash/hash_slinger.h"
-
 #include "cache_backend.h"
 #include "vcli_priv.h"
 #include "vct.h"
@@ -557,7 +555,6 @@ FetchBody(struct worker *wrk, void *priv)
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
 	assert(bo->state == BOS_INVALID);
-	bo->state = BOS_FINISHED;
 
 	/*
 	 * XXX: The busyobj needs a dstat, but it is not obvious which one
@@ -638,8 +635,8 @@ FetchBody(struct worker *wrk, void *priv)
 	if (bo->state == BOS_FAILED) {
 		wrk->stats.fetch_failed++;
 		VDI_CloseFd(&bo->vbc);
+		obj->exp.ttl = -1.;
 		obj->len = 0;
-		HSH_Drop(wrk, &obj);
 	} else {
 		assert(bo->state == BOS_FETCHING);
 
@@ -669,24 +666,19 @@ FetchBody(struct worker *wrk, void *priv)
 			    "Content-Length: %zd", obj->len);
 		}
 
-
 		if (cls)
 			VDI_CloseFd(&bo->vbc);
 		else
 			VDI_RecycleFd(&bo->vbc);
 
-		if (obj->objcore != NULL) {
-			EXP_Insert(obj);
-			AN(obj->objcore->ban);
-			AZ(obj->ws_o->overflow);
-			HSH_Unbusy(&wrk->stats, obj->objcore);
-			obj->objcore->busyobj = NULL;
-			VMB();
-		}
 
 		/* XXX: Atomic assignment, needs volatile/membar ? */
 		bo->state = BOS_FINISHED;
-
+	}
+	if (obj->objcore != NULL) {
+		VMB();
+		obj->objcore->busyobj = NULL;
+		VMB();
 	}
 	bo->stats = NULL;
 	VBO_DerefBusyObj(wrk, &bo);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 09ee9f4..f9217eb 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -328,7 +328,7 @@ HSH_Lookup(struct sess *sp)
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 		assert(oc->objhead == oh);
 
-		if (oc->flags & OC_F_BUSY) {
+		if (oc->flags & OC_F_BUSY || oc->busyobj != NULL) {
 			CHECK_OBJ_ORNULL(oc->busyobj, BUSYOBJ_MAGIC);
 			if (req->hash_ignore_busy)
 				continue;
@@ -379,7 +379,7 @@ HSH_Lookup(struct sess *sp)
 	AZ(req->objcore);
 	if (oc == NULL			/* We found no live object */
 	    && grace_oc != NULL		/* There is a grace candidate */
-	    && (busy_found 		/* Somebody else is already busy */
+	    && (busy_found		/* Somebody else is already busy */
 	    || !VDI_Healthy(req->director, sp))) {
 					/* Or it is impossible to fetch */
 		o = oc_getobj(&wrk->stats, grace_oc);
@@ -431,7 +431,7 @@ HSH_Lookup(struct sess *sp)
 	oc = wrk->nobjcore;
 	wrk->nobjcore = NULL;
 	AN(oc->flags & OC_F_BUSY);
-	oc->refcnt = 1;
+	oc->refcnt = 1;		/* Owned by busyobj */
 	oc->objhead = oh;
 	VTAILQ_INSERT_TAIL(&oh->objcs, oc, list);
 	/* NB: do not deref objhead the new object inherits our reference */
@@ -440,7 +440,7 @@ HSH_Lookup(struct sess *sp)
 	AZ(req->busyobj);
 	req->busyobj = VBO_GetBusyObj(wrk);
 	req->busyobj->vsl->wid = sp->vsl_id;
-	req->busyobj->refcount = 2;	/* One for headers, one for body*/
+	req->busyobj->refcount = 2;	/* One for req, one for FetchBody */
 
 	VRY_Validate(req->vary_b);
 	if (req->vary_l != NULL)



From phk at varnish-cache.org  Tue Mar 20 07:17:34 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Mar 2012 08:17:34 +0100
Subject: [master] ed645d2 Remove busyobj from objcore under oh-mtx
Message-ID: <E1S9tK2-0002T0-Vm@project.varnish-software.com>

commit ed645d241fd54c65bc07acab75ba88efdb4281b9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 20 07:17:21 2012 +0000

    Remove busyobj from objcore under oh-mtx

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index d71a6cb..eb7b855 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -35,10 +35,11 @@
 
 #include "cache.h"
 
+#include "hash/hash_slinger.h"
+
 #include "cache_backend.h"
 #include "vcli_priv.h"
 #include "vct.h"
-#include "vmb.h"
 #include "vtcp.h"
 
 static unsigned fetchfrag;
@@ -675,11 +676,8 @@ FetchBody(struct worker *wrk, void *priv)
 		/* XXX: Atomic assignment, needs volatile/membar ? */
 		bo->state = BOS_FINISHED;
 	}
-	if (obj->objcore != NULL) {
-		VMB();
-		obj->objcore->busyobj = NULL;
-		VMB();
-	}
+	if (obj->objcore != NULL)
+		HSH_Complete(obj->objcore);
 	bo->stats = NULL;
 	VBO_DerefBusyObj(wrk, &bo);
 }
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index f9217eb..e746813 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -566,6 +566,24 @@ HSH_Drop(struct worker *wrk, struct object **oo)
 }
 
 /*---------------------------------------------------------------------
+ * Remove the busyobj from an objcore
+ */
+
+void
+HSH_Complete(struct objcore *oc)
+{
+	struct objhead *oh;
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	oh = oc->objhead;
+	CHECK_OBJ(oh, OBJHEAD_MAGIC);
+
+	Lck_Lock(&oh->mtx);
+	oc->busyobj = NULL;
+	Lck_Unlock(&oh->mtx);
+}
+
+/*---------------------------------------------------------------------
  * Unbusy an objcore when the object is completely fetched.
  */
 
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 628d080..17d67f9 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -94,6 +94,7 @@ struct objhead {
 };
 
 void HSH_Unbusy(struct dstat *, struct objcore *);
+void HSH_Complete(struct objcore *oc);
 void HSH_DeleteObjHead(struct dstat *, struct objhead *oh);
 int HSH_Deref(struct dstat *, struct objcore *oc, struct object **o);
 #endif /* VARNISH_CACHE_CHILD */



From phk at varnish-cache.org  Tue Mar 20 10:57:10 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 20 Mar 2012 11:57:10 +0100
Subject: [master] 4edc10c The busyobj refcount must be protected by the
	oh->mtx just like the oc->busyobj pointer.
Message-ID: <E1S9wkY-0004Fu-Ff@project.varnish-software.com>

commit 4edc10c5d460d38c5a2cecffa572e0001b180ebb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 20 10:56:04 2012 +0000

    The busyobj refcount must be protected by the oh->mtx just like the
    oc->busyobj pointer.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index f337f66..9abf2da 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -725,7 +725,6 @@ double BAN_Time(const struct ban *ban);
 /* cache_busyobj.c */
 void VBO_Init(void);
 struct busyobj *VBO_GetBusyObj(struct worker *wrk);
-void VBO_RefBusyObj(struct busyobj *busyobj);
 void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
 void VBO_Free(struct busyobj **vbo);
 
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 79b55ed..0489734 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -138,20 +138,10 @@ VBO_GetBusyObj(struct worker *wrk)
 }
 
 void
-VBO_RefBusyObj(struct busyobj *bo)
-{
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	Lck_Lock(&bo->mtx);
-	assert(bo->refcount > 0);
-	bo->refcount++;
-	Lck_Unlock(&bo->mtx);
-}
-
-void
 VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 {
 	struct busyobj *bo;
+	struct objcore *oc;
 	unsigned r;
 
 	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
@@ -160,17 +150,28 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 	*pbo = NULL;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_ORNULL(bo->fetch_obj, OBJECT_MAGIC);
-	Lck_Lock(&bo->mtx);
-	assert(bo->refcount > 0);
-	r = --bo->refcount;
-	Lck_Unlock(&bo->mtx);
+	if (bo->fetch_obj != NULL && bo->fetch_obj->objcore != NULL) {
+		oc = bo->fetch_obj->objcore;
+		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+		CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
+		Lck_Lock(&oc->objhead->mtx);
+		assert(bo->refcount > 0);
+		r = --bo->refcount;
+		Lck_Unlock(&oc->objhead->mtx);
+	} else {
+		oc = NULL;
+		Lck_Lock(&bo->mtx);
+		assert(bo->refcount > 0);
+		r = --bo->refcount;
+		Lck_Unlock(&bo->mtx);
+	}
 
 	if (r)
 		return;
 
 	VSL_Flush(bo->vsl, 0);
 
-	if (bo->fetch_obj != NULL && bo->fetch_obj->objcore != NULL) {
+	if (oc != NULL) {
 		AN(wrk);
 		(void)HSH_Deref(&wrk->stats, NULL, &bo->fetch_obj);
 	}



From phk at varnish-cache.org  Thu Mar 22 11:16:53 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Mar 2012 12:16:53 +0100
Subject: [master] 4d33fab Introduce a couple of local busyobj variables for
	clarity
Message-ID: <E1SAg0j-0000a9-D1@project.varnish-software.com>

commit 4d33fabc5e7a5b2820ceaee7c60ee5bc3e8ee656
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 22 11:16:09 2012 +0000

    Introduce a couple of local busyobj variables for clarity

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index a5b058b..6b692f8 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1261,6 +1261,8 @@ XDOT err_pass [label="ERROR",shape=plaintext]
 static int
 cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
 {
+	struct busyobj *bo;
+
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -1270,15 +1272,16 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
 	AZ(req->busyobj);
 
 	req->busyobj = VBO_GetBusyObj(wrk);
-	req->busyobj->vsl->wid = sp->vsl_id;
-	req->busyobj->refcount = 2;
-	http_Setup(req->busyobj->bereq, req->busyobj->ws, req->busyobj->vsl);
+	bo = req->busyobj;
+	bo->vsl->wid = sp->vsl_id;
+	bo->refcount = 2;
+	http_Setup(bo->bereq, bo->ws, bo->vsl);
 	http_FilterReq(sp, HTTPH_R_PASS);
 
 	VCL_pass_method(sp);
 
 	if (req->handling == VCL_RET_ERROR) {
-		http_Teardown(req->busyobj->bereq);
+		http_Teardown(bo->bereq);
 		VBO_DerefBusyObj(wrk, &req->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
@@ -1317,6 +1320,7 @@ DOT err_pipe [label="ERROR",shape=plaintext]
 static int
 cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req)
 {
+	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -1326,8 +1330,9 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req)
 
 	wrk->acct_tmp.pipe++;
 	req->busyobj = VBO_GetBusyObj(wrk);
-	req->busyobj->vsl->wid = sp->vsl_id;
-	http_Setup(req->busyobj->bereq, req->busyobj->ws, req->busyobj->vsl);
+	bo = req->busyobj;
+	bo->vsl->wid = sp->vsl_id;
+	http_Setup(bo->bereq, bo->ws, bo->vsl);
 	http_FilterReq(sp, 0);
 
 	VCL_pipe_method(sp);
@@ -1338,7 +1343,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req)
 
 	PipeSession(sp);
 	assert(WRW_IsReleased(wrk));
-	http_Teardown(req->busyobj->bereq);
+	http_Teardown(bo->bereq);
 	VBO_DerefBusyObj(wrk, &req->busyobj);
 	sp->step = STP_DONE;
 	return (0);



From phk at varnish-cache.org  Thu Mar 22 17:53:04 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Mar 2012 18:53:04 +0100
Subject: [master] ab58597 Add a separate table for the http VSL headers
Message-ID: <E1SAmC8-0001RP-EQ@project.varnish-software.com>

commit ab585970963ddf2c1bfa0882d1f4ac93da1e6d43
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 22 13:24:00 2012 +0000

    Add a separate table for the http VSL headers

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 21ca1df..885e708 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -53,26 +53,17 @@ SLTM(Length)
 
 SLTM(FetchError)
 
-SLTM(RxRequest)
-SLTM(RxResponse)
-SLTM(RxStatus)
-SLTM(RxURL)
-SLTM(RxProtocol)
-SLTM(RxHeader)
+#define SLTH(aa)	SLTM(Rx##aa)
+#include "tbl/vsl_tags_http.h"
+#undef SLTH
 
-SLTM(TxRequest)
-SLTM(TxResponse)
-SLTM(TxStatus)
-SLTM(TxURL)
-SLTM(TxProtocol)
-SLTM(TxHeader)
+#define SLTH(aa)	SLTM(Tx##aa)
+#include "tbl/vsl_tags_http.h"
+#undef SLTH
 
-SLTM(ObjRequest)
-SLTM(ObjResponse)
-SLTM(ObjStatus)
-SLTM(ObjURL)
-SLTM(ObjProtocol)
-SLTM(ObjHeader)
+#define SLTH(aa)	SLTM(Obj##aa)
+#include "tbl/vsl_tags_http.h"
+#undef SLTH
 
 SLTM(LostHeader)
 
diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h
new file mode 100644
index 0000000..5e8544f
--- /dev/null
+++ b/include/tbl/vsl_tags_http.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2012 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Define the VSL tags for HTTP protocol messages
+ *
+ */
+
+SLTH(Request)
+SLTH(Response)
+SLTH(Status)
+SLTH(URL)
+SLTH(Protocol)
+SLTH(Header)
+SLTH(Lost)



From phk at varnish-cache.org  Thu Mar 22 17:53:04 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Mar 2012 18:53:04 +0100
Subject: [master] e4996ef Link the HTTP VSLs to the index in the http structure
Message-ID: <E1SAmC8-0001RU-FT@project.varnish-software.com>

commit e4996efa99d8a0407d6761fce47ca08968d4cb3d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 22 14:05:30 2012 +0000

    Link the HTTP VSLs to the index in the http structure

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 885e708..ecf0a20 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -53,15 +53,15 @@ SLTM(Length)
 
 SLTM(FetchError)
 
-#define SLTH(aa)	SLTM(Rx##aa)
+#define SLTH(aa, bb)	SLTM(Rx##aa)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa)	SLTM(Tx##aa)
+#define SLTH(aa, bb)	SLTM(Tx##aa)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa)	SLTM(Obj##aa)
+#define SLTH(aa, bb)	SLTM(Obj##aa)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h
index 5e8544f..542af07 100644
--- a/include/tbl/vsl_tags_http.h
+++ b/include/tbl/vsl_tags_http.h
@@ -27,12 +27,15 @@
  *
  * Define the VSL tags for HTTP protocol messages
  *
+ * The order of this table is not random, do not resort.
+ * In particular, the FIRST and LOST entries must be last, in that order.
+ *
  */
 
-SLTH(Request)
-SLTH(Response)
-SLTH(Status)
-SLTH(URL)
-SLTH(Protocol)
-SLTH(Header)
-SLTH(Lost)
+SLTH(Request,	HTTP_HDR_REQ)
+SLTH(URL,	HTTP_HDR_URL)
+SLTH(Protocol,	HTTP_HDR_PROTO)
+SLTH(Status,	HTTP_HDR_STATUS)
+SLTH(Response,	HTTP_HDR_RESPONSE)
+SLTH(Header,	HTTP_HDR_FIRST)
+SLTH(Lost,	HTTP_HDR_LOST)



From phk at varnish-cache.org  Thu Mar 22 17:53:04 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Mar 2012 18:53:04 +0100
Subject: [master] 61367de Simplify the CPP magic involved in HTTP headers,
	now that we can rely on everybody agreeing about the order of these
	fields.
Message-ID: <E1SAmC8-0001Ra-Ge@project.varnish-software.com>

commit 61367de4084f4c806944c989c021535bb5cf7e16
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 22 14:06:52 2012 +0000

    Simplify the CPP magic involved in HTTP headers, now that we can rely
    on everybody agreeing about the order of these fields.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 9abf2da..ec385c5 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -82,13 +82,9 @@ body_status(enum body_status e)
 
 enum {
 	/* Fields from the first line of HTTP proto */
-	HTTP_HDR_REQ,
-	HTTP_HDR_URL,
-	HTTP_HDR_PROTO,
-	HTTP_HDR_STATUS,
-	HTTP_HDR_RESPONSE,
-	/* HTTP header lines */
-	HTTP_HDR_FIRST,
+#define SLTH(aa, bb)	bb,
+#include "tbl/vsl_tags_http.h"
+#undef SLTH
 };
 
 struct SHA256Context;
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 4d2a9fa..bcde416 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -41,24 +41,11 @@
 #include "tbl/http_headers.h"
 #undef HTTPH
 
-/*lint -save -e773 not () */
-#define LOGMTX2(ax, bx, cx)	[bx] = SLT_##ax##cx
-
-#define LOGMTX1(ax) {					\
-	LOGMTX2(ax, HTTP_HDR_REQ,	Request),	\
-	LOGMTX2(ax, HTTP_HDR_RESPONSE,	Response),	\
-	LOGMTX2(ax, HTTP_HDR_STATUS,	Status),	\
-	LOGMTX2(ax, HTTP_HDR_URL,	URL),		\
-	LOGMTX2(ax, HTTP_HDR_PROTO,	Protocol),	\
-	LOGMTX2(ax, HTTP_HDR_FIRST,	Header),	\
-	}
-
-static const enum VSL_tag_e logmtx[][HTTP_HDR_FIRST + 1] = {
-	[HTTP_Rx] = LOGMTX1(Rx),
-	[HTTP_Tx] = LOGMTX1(Tx),
-	[HTTP_Obj] = LOGMTX1(Obj)
+static const enum VSL_tag_e foo[] = {
+	[HTTP_Rx] = SLT_RxRequest,
+	[HTTP_Tx] = SLT_TxRequest,
+	[HTTP_Obj] = SLT_ObjRequest,
 };
-/*lint -restore */
 
 static enum VSL_tag_e
 http2shmlog(const struct http *hp, int t)
@@ -69,7 +56,7 @@ http2shmlog(const struct http *hp, int t)
 		t = HTTP_HDR_FIRST;
 	assert(hp->logtag >= HTTP_Rx && hp->logtag <= HTTP_Obj); /*lint !e685*/
 	assert(t >= HTTP_HDR_REQ && t <= HTTP_HDR_FIRST);
-	return (logmtx[hp->logtag][t]);
+	return ((enum VSL_tag_e)(foo[hp->logtag] + t));
 }
 
 static void



From phk at varnish-cache.org  Thu Mar 22 17:53:04 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 22 Mar 2012 18:53:04 +0100
Subject: [master] f177a35 Rework the way we log HTTP headers: Rather than the
	Rx/Tx/Obj, tag them with Req/Resp/Bereq/Beresp/Obj instead.
Message-ID: <E1SAmC8-0001Rh-IB@project.varnish-software.com>

commit f177a357f792e116f9cc3ba4c64dfc5df3735ae5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Mar 22 14:33:34 2012 +0000

    Rework the way we log HTTP headers:  Rather than the Rx/Tx/Obj, tag them
    with Req/Resp/Bereq/Beresp/Obj instead.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ec385c5..0bb75df 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -151,9 +151,11 @@ struct ws {
  */
 
 enum httpwhence {
-	HTTP_Rx	 = 1,
-	HTTP_Tx  = 2,
-	HTTP_Obj = 3
+	HTTP_Req	= 1,
+	HTTP_Resp,
+	HTTP_Bereq,
+	HTTP_Beresp,
+	HTTP_Obj
 };
 
 /* NB: remember to update http_Copy() if you add fields */
@@ -810,7 +812,7 @@ void http_PrintfHeader(struct http *to, const char *fmt, ...)
 void http_SetHeader(struct http *to, const char *hdr);
 void http_SetH(const struct http *to, unsigned n, const char *fm);
 void http_ForceGet(const struct http *to);
-void http_Setup(struct http *ht, struct ws *ws, struct vsl_log *);
+void HTTP_Setup(struct http *, struct ws *, struct vsl_log *, enum httpwhence);
 void http_Teardown(struct http *ht);
 int http_GetHdr(const struct http *hp, const char *hdr, char **ptr);
 int http_GetHdrData(const struct http *hp, const char *hdr,
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 6b692f8..7104e0f 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -275,7 +275,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 			req->obj->last_lru = req->t_resp;
 		req->obj->last_use = req->t_resp;	/* XXX: locking ? */
 	}
-	http_Setup(req->resp, req->ws, req->vsl);
+	HTTP_Setup(req->resp, req->ws, req->vsl, HTTP_Resp);
 	RES_BuildHttp(sp);
 	VCL_deliver_method(sp);
 	switch (req->handling) {
@@ -577,7 +577,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 	AZ(bo->should_close);
 	AZ(req->storage_hint);
 
-	http_Setup(bo->beresp, bo->ws, bo->vsl);
+	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
 
 	need_host_hdr = !http_GetHdr(bo->bereq, H_Host, NULL);
 
@@ -1197,7 +1197,7 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AZ(req->obj);
 
-	http_Setup(bo->bereq, bo->ws, bo->vsl);
+	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
 	http_FilterReq(sp, HTTPH_R_FETCH);
 	http_ForceGet(bo->bereq);
 	if (cache_param->http_gzip_support) {
@@ -1275,7 +1275,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
 	bo = req->busyobj;
 	bo->vsl->wid = sp->vsl_id;
 	bo->refcount = 2;
-	http_Setup(bo->bereq, bo->ws, bo->vsl);
+	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
 	http_FilterReq(sp, HTTPH_R_PASS);
 
 	VCL_pass_method(sp);
@@ -1332,7 +1332,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req)
 	req->busyobj = VBO_GetBusyObj(wrk);
 	bo = req->busyobj;
 	bo->vsl->wid = sp->vsl_id;
-	http_Setup(bo->bereq, bo->ws, bo->vsl);
+	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, HTTP_Bereq);
 	http_FilterReq(sp, 0);
 
 	VCL_pipe_method(sp);
@@ -1508,7 +1508,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 
 	EXP_Clr(&req->exp);
 
-	http_Setup(req->http, req->ws, req->vsl);
+	HTTP_Setup(req->http, req->ws, req->vsl, HTTP_Req);
 	req->err_code = http_DissectRequest(sp);
 
 	/* If we could not even parse the request, just close */
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index bcde416..0380469 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -42,9 +42,11 @@
 #undef HTTPH
 
 static const enum VSL_tag_e foo[] = {
-	[HTTP_Rx] = SLT_RxRequest,
-	[HTTP_Tx] = SLT_TxRequest,
-	[HTTP_Obj] = SLT_ObjRequest,
+	[HTTP_Req]	= SLT_ReqRequest,
+	[HTTP_Resp]	= SLT_RespRequest,
+	[HTTP_Bereq]	= SLT_BereqRequest,
+	[HTTP_Beresp]	= SLT_BerespRequest,
+	[HTTP_Obj]	= SLT_ObjRequest,
 };
 
 static enum VSL_tag_e
@@ -54,7 +56,7 @@ http2shmlog(const struct http *hp, int t)
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	if (t > HTTP_HDR_FIRST)
 		t = HTTP_HDR_FIRST;
-	assert(hp->logtag >= HTTP_Rx && hp->logtag <= HTTP_Obj); /*lint !e685*/
+	assert(hp->logtag >= HTTP_Req && hp->logtag <= HTTP_Obj); /*lint !e685*/
 	assert(t >= HTTP_HDR_REQ && t <= HTTP_HDR_FIRST);
 	return ((enum VSL_tag_e)(foo[hp->logtag] + t));
 }
@@ -118,9 +120,11 @@ HTTP_create(void *p, uint16_t nhttp)
 /*--------------------------------------------------------------------*/
 
 void
-http_Setup(struct http *hp, struct ws *ws, struct vsl_log *vsl)
+HTTP_Setup(struct http *hp, struct ws *ws, struct vsl_log *vsl,
+    enum httpwhence  whence)
 {
 	http_Teardown(hp);
+	hp->logtag = whence;
 	hp->ws = ws;
 	hp->vsl = vsl;
 }
@@ -659,8 +663,6 @@ http_DissectRequest(const struct sess *sp)
 	hp = sp->req->http;
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 
-	hp->logtag = HTTP_Rx;
-
 	retval = http_splitline(hp, htc,
 	    HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO);
 	if (retval != 0) {
@@ -683,7 +685,6 @@ http_DissectResponse(struct http *hp, const struct http_conn *htc)
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-	hp->logtag = HTTP_Rx;
 
 	if (http_splitline(hp, htc,
 	    HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE))
@@ -843,7 +844,6 @@ http_FilterReq(const struct sess *sp, unsigned how)
 
 	hp = sp->req->busyobj->bereq;
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-	hp->logtag = HTTP_Tx;
 
 	http_linkh(hp, sp->req->http, HTTP_HDR_REQ);
 	http_linkh(hp, sp->req->http, HTTP_HDR_URL);
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 1e065ba..8579e37 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -115,7 +115,6 @@ RES_BuildHttp(const struct sess *sp)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	http_ClrHeader(req->resp);
-	req->resp->logtag = HTTP_Tx;
 	http_FilterResp(req->obj->http, req->resp, 0);
 
 	if (!(req->res_mode & RES_LEN)) {
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 9f013d2..b71401d 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -256,7 +256,7 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo, struct objcore **ocp,
 	WS_Assert(o->ws_o);
 	assert(o->ws_o->e <= (char*)ptr + ltot);
 
-	http_Setup(o->http, o->ws_o, bo->vsl);
+	HTTP_Setup(o->http, o->ws_o, bo->vsl, HTTP_Obj);
 	o->http->magic = HTTP_MAGIC;
 	o->exp = bo->exp;
 	VTAILQ_INIT(&o->store);
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index ecf0a20..1321493 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -53,11 +53,19 @@ SLTM(Length)
 
 SLTM(FetchError)
 
-#define SLTH(aa, bb)	SLTM(Rx##aa)
+#define SLTH(aa, bb)	SLTM(Req##aa)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa, bb)	SLTM(Tx##aa)
+#define SLTH(aa, bb)	SLTM(Resp##aa)
+#include "tbl/vsl_tags_http.h"
+#undef SLTH
+
+#define SLTH(aa, bb)	SLTM(Bereq##aa)
+#include "tbl/vsl_tags_http.h"
+#undef SLTH
+
+#define SLTH(aa, bb)	SLTM(Beresp##aa)
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 



From phk at varnish-cache.org  Fri Mar 23 10:44:10 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 23 Mar 2012 11:44:10 +0100
Subject: [master] 745c828 Add two fields to the VSL tag definition,
	for documentation use.
Message-ID: <E1SB1yc-0004kk-Hz@project.varnish-software.com>

commit 745c8287c4c3c3fa1ab2f93a5cc571a622413c16
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Mar 23 10:43:47 2012 +0000

    Add two fields to the VSL tag definition, for documentation use.

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 1321493..a559eed 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -34,69 +34,74 @@
  *
  * XXX: Please add new entries a the end to not break saved log-segments.
  * XXX: we can resort them when we have a major release.
+ *
+ * Arguments:
+ *	Tag-Name
+ *	Short Description (1 line, max ?? chars)
+ *	Long Description (Multi line)
  */
 
-SLTM(Debug)
-SLTM(Error)
-SLTM(CLI)
-SLTM(StatSess)
-SLTM(ReqEnd)
-SLTM(SessionOpen)
-SLTM(SessionClose)
-SLTM(BackendOpen)
-SLTM(BackendXID)
-SLTM(BackendReuse)
-SLTM(BackendClose)
-SLTM(HttpGarbage)
-SLTM(Backend)
-SLTM(Length)
+SLTM(Debug, "", "")
+SLTM(Error, "", "")
+SLTM(CLI, "", "")
+SLTM(StatSess, "", "")
+SLTM(ReqEnd, "", "")
+SLTM(SessionOpen, "", "")
+SLTM(SessionClose, "", "")
+SLTM(BackendOpen, "", "")
+SLTM(BackendXID, "", "")
+SLTM(BackendReuse, "", "")
+SLTM(BackendClose, "", "")
+SLTM(HttpGarbage, "", "")
+SLTM(Backend, "", "")
+SLTM(Length, "", "")
 
-SLTM(FetchError)
+SLTM(FetchError, "", "")
 
-#define SLTH(aa, bb)	SLTM(Req##aa)
+#define SLTH(aa, bb)	SLTM(Req##aa, "", "")
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa, bb)	SLTM(Resp##aa)
+#define SLTH(aa, bb)	SLTM(Resp##aa, "", "")
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa, bb)	SLTM(Bereq##aa)
+#define SLTH(aa, bb)	SLTM(Bereq##aa, "", "")
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa, bb)	SLTM(Beresp##aa)
+#define SLTH(aa, bb)	SLTM(Beresp##aa, "", "")
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-#define SLTH(aa, bb)	SLTM(Obj##aa)
+#define SLTH(aa, bb)	SLTM(Obj##aa, "", "")
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
 
-SLTM(LostHeader)
+SLTM(LostHeader, "", "")
 
-SLTM(TTL)
-SLTM(Fetch_Body)
-SLTM(VCL_acl)
-SLTM(VCL_call)
-SLTM(VCL_trace)
-SLTM(VCL_return)
-SLTM(VCL_error)
-SLTM(ReqStart)
-SLTM(Hit)
-SLTM(HitPass)
-SLTM(ExpBan)
-SLTM(ExpKill)
-SLTM(WorkThread)
+SLTM(TTL, "", "")
+SLTM(Fetch_Body, "", "")
+SLTM(VCL_acl, "", "")
+SLTM(VCL_call, "", "")
+SLTM(VCL_trace, "", "")
+SLTM(VCL_return, "", "")
+SLTM(VCL_error, "", "")
+SLTM(ReqStart, "", "")
+SLTM(Hit, "", "")
+SLTM(HitPass, "", "")
+SLTM(ExpBan, "", "")
+SLTM(ExpKill, "", "")
+SLTM(WorkThread, "", "")
 
-SLTM(ESI_xmlerror)
+SLTM(ESI_xmlerror, "", "")
 
-SLTM(Hash)
+SLTM(Hash, "", "")
 
-SLTM(Backend_health)
+SLTM(Backend_health, "", "")
 
-SLTM(VCL_Debug)
-SLTM(VCL_Log)
-SLTM(VCL_Error)
+SLTM(VCL_Debug, "", "")
+SLTM(VCL_Log, "", "")
+SLTM(VCL_Error, "", "")
 
-SLTM(Gzip)
+SLTM(Gzip, "", "")
diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h
index 8176cce..6f192e9 100644
--- a/include/vapi/vsl_int.h
+++ b/include/vapi/vsl_int.h
@@ -73,7 +73,7 @@
  */
 enum VSL_tag_e {
 	SLT_Bogus = 0,
-#define SLTM(foo)	SLT_##foo,
+#define SLTM(foo,sdesc,ldesc)	SLT_##foo,
 #include "tbl/vsl_tags.h"
 #undef SLTM
 	SLT_Reserved = 255
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 8e7001a..cd6034d 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -55,7 +55,7 @@
 /*--------------------------------------------------------------------*/
 
 const char *VSL_tags[256] = {
-#  define SLTM(foo)       [SLT_##foo] = #foo,
+#  define SLTM(foo,sdesc,ldesc)       [SLT_##foo] = #foo,
 #  include "tbl/vsl_tags.h"
 #  undef SLTM
 };



From apj at varnish-cache.org  Mon Mar 26 07:31:02 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Mon, 26 Mar 2012 09:31:02 +0200
Subject: [master] 9fc9e7d Add short descriptions
Message-ID: <E1SC4OM-0002Lu-62@project.varnish-software.com>

commit 9fc9e7d0d498e435ee27913d7f5714000fc82ad5
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Mar 26 09:30:35 2012 +0200

    Add short descriptions

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index a559eed..86c9cc2 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -43,20 +43,20 @@
 
 SLTM(Debug, "", "")
 SLTM(Error, "", "")
-SLTM(CLI, "", "")
-SLTM(StatSess, "", "")
-SLTM(ReqEnd, "", "")
-SLTM(SessionOpen, "", "")
-SLTM(SessionClose, "", "")
-SLTM(BackendOpen, "", "")
-SLTM(BackendXID, "", "")
-SLTM(BackendReuse, "", "")
-SLTM(BackendClose, "", "")
+SLTM(CLI, "CLI communication", "CLI communication between master and child process.")
+SLTM(StatSess, "Session statistics", "")
+SLTM(ReqEnd, "Client request end", "")
+SLTM(SessionOpen, "Client connection opened", "")
+SLTM(SessionClose, "Client connection closed", "")
+SLTM(BackendOpen, "Backend connection opened", "")
+SLTM(BackendXID, "The unique ID of the backend transaction", "")
+SLTM(BackendReuse, "Backend connection reused", "")
+SLTM(BackendClose, "Backend connection closed", "")
 SLTM(HttpGarbage, "", "")
-SLTM(Backend, "", "")
-SLTM(Length, "", "")
+SLTM(Backend, "Backend selected", "")
+SLTM(Length, "Size of object body", "")
 
-SLTM(FetchError, "", "")
+SLTM(FetchError, "Error while fetching object", "")
 
 #define SLTH(aa, bb)	SLTM(Req##aa, "", "")
 #include "tbl/vsl_tags_http.h"
@@ -80,28 +80,28 @@ SLTM(FetchError, "", "")
 
 SLTM(LostHeader, "", "")
 
-SLTM(TTL, "", "")
-SLTM(Fetch_Body, "", "")
+SLTM(TTL, "TTL set on object", "")
+SLTM(Fetch_Body, "Body fetched from backend", "")
 SLTM(VCL_acl, "", "")
-SLTM(VCL_call, "", "")
-SLTM(VCL_trace, "", "")
-SLTM(VCL_return, "", "")
-SLTM(VCL_error, "", "")
-SLTM(ReqStart, "", "")
-SLTM(Hit, "", "")
-SLTM(HitPass, "", "")
-SLTM(ExpBan, "", "")
-SLTM(ExpKill, "", "")
+SLTM(VCL_call, "VCL method called", "")
+SLTM(VCL_trace, "VCL trace data", "")
+SLTM(VCL_return, "VCL method return value", "")
+SLTM(VCL_error, "Unused", "")
+SLTM(ReqStart, "Client request start", "")
+SLTM(Hit, "Hit object in cache", "")
+SLTM(HitPass, "Hit for pass object in cache", "")
+SLTM(ExpBan, "Object evicted due to ban", "")
+SLTM(ExpKill, "Object expired", "")
 SLTM(WorkThread, "", "")
 
-SLTM(ESI_xmlerror, "", "")
+SLTM(ESI_xmlerror, "Error while parsing ESI tags", "")
 
-SLTM(Hash, "", "")
+SLTM(Hash, "Value added to hash", "")
 
-SLTM(Backend_health, "", "")
+SLTM(Backend_health, "Backend health check", "")
 
-SLTM(VCL_Debug, "", "")
-SLTM(VCL_Log, "", "")
+SLTM(VCL_Debug, "Unused", "")
+SLTM(VCL_Log, "Log statement from VCL", "")
 SLTM(VCL_Error, "", "")
 
-SLTM(Gzip, "", "")
+SLTM(Gzip, "G(un)zip performed on object", "")



From tfheen at varnish-cache.org  Mon Mar 26 08:13:56 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 26 Mar 2012 10:13:56 +0200
Subject: [master] e37585d Make sure we distribute vsl_tags_http.h too
Message-ID: <E1SC53s-0005s0-SP@project.varnish-software.com>

commit e37585df5d519a16f3755282a1b48e41f07bc2f5
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Mar 23 15:18:27 2012 +0100

    Make sure we distribute vsl_tags_http.h too

diff --git a/include/Makefile.am b/include/Makefile.am
index bb00ce6..d40169b 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -17,6 +17,7 @@ pkginclude_HEADERS = \
 	tbl/vsc_fields.h \
 	tbl/vsc_f_main.h \
 	tbl/vsl_tags.h \
+	tbl/vsl_tags_http.h \
 	vapi/vsm.h \
 	vapi/vsm_int.h \
 	vapi/vsc.h \



From tfheen at varnish-cache.org  Mon Mar 26 08:13:56 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 26 Mar 2012 10:13:56 +0200
Subject: [master] 5a1b770 Adjust varnishncsa and varnishreplay for the new tag
	names
Message-ID: <E1SC53s-0005s5-Tk@project.varnish-software.com>

commit 5a1b7707dad90e0a5ff23032c666f8502cf46e5f
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Mar 23 15:20:51 2012 +0100

    Adjust varnishncsa and varnishreplay for the new tag names

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 9abb9b6..9bdf87f 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -293,7 +293,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 			lp->df_h = trimfield(ptr, end);
 		break;
 
-	case SLT_TxRequest:
+	case SLT_BereqRequest:
 		if (!lp->active)
 			break;
 		if (lp->df_m != NULL) {
@@ -303,7 +303,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		lp->df_m = trimline(ptr, end);
 		break;
 
-	case SLT_TxURL: {
+	case SLT_BereqURL: {
 		char *qs;
 
 		if (!lp->active)
@@ -322,7 +322,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		break;
 	}
 
-	case SLT_TxProtocol:
+	case SLT_BereqProtocol:
 		if (!lp->active)
 			break;
 		if (lp->df_H != NULL) {
@@ -332,7 +332,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		lp->df_H = trimline(ptr, end);
 		break;
 
-	case SLT_RxStatus:
+	case SLT_BerespStatus:
 		if (!lp->active)
 			break;
 		if (lp->df_s != NULL) {
@@ -342,7 +342,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		lp->df_s = trimline(ptr, end);
 		break;
 
-	case SLT_RxHeader:
+	case SLT_BerespHeader:
 		if (!lp->active)
 			break;
 		if (isprefix(ptr, "content-length:", end, &next))
@@ -353,7 +353,7 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		}
 		break;
 
-	case SLT_TxHeader:
+	case SLT_BereqHeader:
 		if (!lp->active)
 			break;
 		split = strchr(ptr, ':');
@@ -412,7 +412,7 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		lp->df_h = trimfield(ptr, end);
 		break;
 
-	case SLT_RxRequest:
+	case SLT_ReqRequest:
 		if (!lp->active)
 			break;
 		if (lp->df_m != NULL) {
@@ -422,7 +422,7 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		lp->df_m = trimline(ptr, end);
 		break;
 
-	case SLT_RxURL: {
+	case SLT_ReqURL: {
 		char *qs;
 
 		if (!lp->active)
@@ -441,7 +441,7 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		break;
 	}
 
-	case SLT_RxProtocol:
+	case SLT_ReqProtocol:
 		if (!lp->active)
 			break;
 		if (lp->df_H != NULL) {
@@ -451,7 +451,7 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		lp->df_H = trimline(ptr, end);
 		break;
 
-	case SLT_TxStatus:
+	case SLT_ObjStatus:
 		if (!lp->active)
 			break;
 		if (lp->df_s != NULL)
@@ -460,14 +460,14 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 			lp->df_s = trimline(ptr, end);
 		break;
 
-	case SLT_TxHeader:
-	case SLT_RxHeader:
+	case SLT_ObjHeader:
+	case SLT_ReqHeader:
 		if (!lp->active)
 			break;
 		split = strchr(ptr, ':');
 		if (split == NULL)
 			break;
-		if (tag == SLT_RxHeader &&
+		if (tag == SLT_ReqHeader &&
 		    isprefix(ptr, "authorization:", end, &next) &&
 		    isprefix(next, "basic", end, &next)) {
 			free(lp->df_u);
@@ -479,7 +479,7 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 			AN(split);
 			h->key = trimline(ptr, split);
 			h->value = trimline(split+1, end);
-			if (tag == SLT_RxHeader)
+			if (tag == SLT_ReqHeader)
 				VTAILQ_INSERT_HEAD(&lp->req_headers, h, list);
 			else
 				VTAILQ_INSERT_HEAD(&lp->resp_headers, h, list);
diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c
index 02ad83e..aec3f5f 100644
--- a/bin/varnishreplay/varnishreplay.c
+++ b/bin/varnishreplay/varnishreplay.c
@@ -515,28 +515,28 @@ replay_thread(void *arg)
 		thread_log(2, 0, "%s(%s)", VSL_tags[tag], msg->ptr);
 
 		switch (tag) {
-		case SLT_RxRequest:
+		case SLT_ReqRequest:
 			if (thr->method != NULL)
 				thr->bogus = 1;
 			else
 				thr->method = trimline(thr, ptr);
 			break;
 
-		case SLT_RxURL:
+		case SLT_ReqURL:
 			if (thr->url != NULL)
 				thr->bogus = 1;
 			else
 				thr->url = trimline(thr, ptr);
 			break;
 
-		case SLT_RxProtocol:
+		case SLT_ReqProtocol:
 			if (thr->proto != NULL)
 				thr->bogus = 1;
 			else
 				thr->proto = trimline(thr, ptr);
 			break;
 
-		case SLT_RxHeader:
+		case SLT_ReqHeader:
 			if (thr->nhdr >= sizeof thr->hdr / sizeof *thr->hdr) {
 				thr->bogus = 1;
 			} else {



From apj at varnish-cache.org  Mon Mar 26 08:25:01 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Mon, 26 Mar 2012 10:25:01 +0200
Subject: [master] fc8678c Remove unused tag
Message-ID: <E1SC5Eb-0006kA-4g@project.varnish-software.com>

commit fc8678c77d60575303b13011856a7b44521b312a
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Mar 26 10:24:47 2012 +0200

    Remove unused tag

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 86c9cc2..1c404cf 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -86,7 +86,6 @@ SLTM(VCL_acl, "", "")
 SLTM(VCL_call, "VCL method called", "")
 SLTM(VCL_trace, "VCL trace data", "")
 SLTM(VCL_return, "VCL method return value", "")
-SLTM(VCL_error, "Unused", "")
 SLTM(ReqStart, "Client request start", "")
 SLTM(Hit, "Hit object in cache", "")
 SLTM(HitPass, "Hit for pass object in cache", "")



From phk at varnish-cache.org  Tue Mar 27 11:32:13 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 27 Mar 2012 13:32:13 +0200
Subject: [master] c5a1c75 With streaming,
	we cannot allow a storage chunk to move once we have started to fill
	it. Add a param to the STV_trim() method to indicate
	acceptability of move.
Message-ID: <E1SCUdJ-0005qK-6B@project.varnish-software.com>

commit c5a1c750982d6a758715b1ad69ea9373521ea6e7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 27 08:09:39 2012 +0000

    With streaming, we cannot allow a storage chunk to move once we
    have started to fill it.  Add a param to the STV_trim() method
    to indicate acceptability of move.
    
    The only reason I don't remove STV_trim() entirely, is that a site
    with gazillions very small objects, will want to disable streaming
    and use malloc to squeeze as many objects into memory as possible.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 0bb75df..8f9b6d9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1013,7 +1013,7 @@ int RFC2616_Do_Cond(const struct sess *sp);
 struct object *STV_NewObject(struct busyobj *, struct objcore **,
     const char *hint, unsigned len, uint16_t nhttp);
 struct storage *STV_alloc(struct busyobj *, size_t size);
-void STV_trim(struct storage *st, size_t size);
+void STV_trim(struct storage *st, size_t size, int move_ok);
 void STV_free(struct storage *st);
 void STV_open(void);
 void STV_close(void);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index eb7b855..0246f1b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -195,7 +195,7 @@ vfp_nop_end(struct busyobj *bo)
 		return (0);
 	}
 	if (st->len < st->space)
-		STV_trim(st, st->len);
+		STV_trim(st, st->len, 1);
 	return (0);
 }
 
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index b71401d..f1cbe94 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -391,13 +391,13 @@ STV_alloc(struct busyobj *bo, size_t size)
 }
 
 void
-STV_trim(struct storage *st, size_t size)
+STV_trim(struct storage *st, size_t size, int move_ok)
 {
 
 	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 	AN(st->stevedore);
 	if (st->stevedore->trim)
-		st->stevedore->trim(st, size);
+		st->stevedore->trim(st, size, move_ok);
 }
 
 void
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index 6994314..aae9186 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -42,7 +42,7 @@ struct lru;
 typedef void storage_init_f(struct stevedore *, int ac, char * const *av);
 typedef void storage_open_f(const struct stevedore *);
 typedef struct storage *storage_alloc_f(struct stevedore *, size_t size);
-typedef void storage_trim_f(struct storage *, size_t size);
+typedef void storage_trim_f(struct storage *, size_t size, int move_ok);
 typedef void storage_free_f(struct storage *);
 typedef struct object *storage_allocobj_f(struct stevedore *, struct busyobj *,
     struct objcore **, unsigned ltot, const struct stv_objsecrets *);
diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index 680b870..3d422fe 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -488,7 +488,7 @@ smf_alloc(struct stevedore *st, size_t size)
 /*--------------------------------------------------------------------*/
 
 static void
-smf_trim(struct storage *s, size_t size)
+smf_trim(struct storage *s, size_t size, int move_ok)
 {
 	struct smf *smf;
 	struct smf_sc *sc;
@@ -499,6 +499,10 @@ smf_trim(struct storage *s, size_t size)
 	xxxassert(size > 0);	/* XXX: seen */
 	CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC);
 	assert(size <= smf->size);
+
+	if (!move_ok)
+		return;		/* XXX: trim_smf needs fixed */
+
 	sc = smf->sc;
 	size += (sc->pagesize - 1);
 	size &= ~(sc->pagesize - 1);
diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c
index 2f34c98..0470093 100644
--- a/bin/varnishd/storage/storage_malloc.c
+++ b/bin/varnishd/storage/storage_malloc.c
@@ -145,7 +145,7 @@ sma_free(struct storage *s)
 }
 
 static void
-sma_trim(struct storage *s, size_t size)
+sma_trim(struct storage *s, size_t size, int move_ok)
 {
 	struct sma_sc *sma_sc;
 	struct sma *sma;
@@ -158,6 +158,10 @@ sma_trim(struct storage *s, size_t size)
 
 	assert(sma->sz == sma->s.space);
 	assert(size < sma->sz);
+
+	if (!move_ok)
+		return;
+
 	delta = sma->sz - size;
 	if (delta < 256)
 		return;
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 35845f0..d206a86 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -523,19 +523,6 @@ smp_alloc(struct stevedore *st, size_t size)
 }
 
 /*--------------------------------------------------------------------
- * Trim a bite
- * XXX: We could trim the last allocation.
- */
-
-static void
-smp_trim(struct storage *ss, size_t size)
-{
-
-	(void)ss;
-	(void)size;
-}
-
-/*--------------------------------------------------------------------
  * We don't track frees of storage, we track the objects which own the
  * storage and when there are no more objects in in the first segment,
  * it can be reclaimed.
@@ -562,7 +549,6 @@ const struct stevedore smp_stevedore = {
 	.alloc	=	smp_alloc,
 	.allocobj =	smp_allocobj,
 	.free	=	smp_free,
-	.trim	=	smp_trim,
 };
 
 /*--------------------------------------------------------------------



From phk at varnish-cache.org  Tue Mar 27 11:32:13 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 27 Mar 2012 13:32:13 +0200
Subject: [master] 3a4edf1 Centralize the updating of the fetch_obj->len field.
Message-ID: <E1SCUdJ-0005qQ-7c@project.varnish-software.com>

commit 3a4edf1da8924e3bbfe3d9a1f87e34b5e57ea506
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Mar 27 08:30:51 2012 +0000

    Centralize the updating of the fetch_obj->len field.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8f9b6d9..073eb75 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -216,6 +216,8 @@ struct dstat {
 
 /* Fetch processors --------------------------------------------------*/
 
+void VFP_update_length(const struct busyobj *, ssize_t);
+
 typedef void vfp_begin_f(struct busyobj *, size_t );
 typedef int vfp_bytes_f(struct busyobj *, struct http_conn *, ssize_t);
 typedef int vfp_end_f(struct busyobj *);
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 6699117..c9cc689 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -103,7 +103,7 @@ vfp_esi_bytes_uu(struct busyobj *bo, const struct vef_priv *vef,
 			return (wl);
 		VEP_Parse(bo, (const char *)st->ptr + st->len, wl);
 		st->len += wl;
-		bo->fetch_obj->len += wl;
+		VFP_update_length(bo, wl);
 		bytes -= wl;
 	}
 	return (1);
@@ -140,7 +140,7 @@ vfp_esi_bytes_gu(struct busyobj *bo, const struct vef_priv *vef,
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		xxxassert(i == VGZ_OK || i == VGZ_END);
 		VEP_Parse(bo, dp, dl);
-		bo->fetch_obj->len += dl;
+		VFP_update_length(bo, dl);
 	}
 	return (1);
 }
@@ -217,7 +217,7 @@ vfp_vep_callback(struct busyobj *bo, ssize_t l, enum vgz_flag flg)
 		}
 		i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
 		vef->tot += dl;
-		bo->fetch_obj->len += dl;
+		VFP_update_length(bo, dl);
 	} while (!VGZ_IbufEmpty(vef->vgz) ||
 	    (flg != VGZ_NORMAL && VGZ_ObufFull(vef->vgz)));
 	assert(VGZ_IbufEmpty(vef->vgz));
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0246f1b..e019702 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -113,6 +113,17 @@ VFP_End(struct busyobj *bo)
 		assert(bo->state == BOS_FAILED);
 }
 
+void
+VFP_update_length(const struct busyobj *bo, ssize_t l)
+{
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->fetch_obj, OBJECT_MAGIC);
+	if (l == 0)
+		return;
+	assert(l > 0);
+	bo->fetch_obj->len += l;
+}
 
 /*--------------------------------------------------------------------
  * VFP_NOP
@@ -165,7 +176,7 @@ vfp_nop_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 		if (wl <= 0)
 			return (wl);
 		st->len += wl;
-		bo->fetch_obj->len += wl;
+		VFP_update_length(bo, wl);
 		bytes -= wl;
 	}
 	return (1);
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 343bd35..ca40903 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -468,7 +468,7 @@ vfp_gunzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		if (i != VGZ_OK && i != VGZ_END)
 			return(FetchError(bo, "Gunzip data error"));
-		bo->fetch_obj->len += dl;
+		VFP_update_length(bo, dl);
 	}
 	assert(i == Z_OK || i == Z_STREAM_END);
 	return (1);
@@ -543,7 +543,7 @@ vfp_gzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
-		bo->fetch_obj->len += dl;
+		VFP_update_length(bo, dl);
 	}
 	return (1);
 }
@@ -569,7 +569,7 @@ vfp_gzip_end(struct busyobj *bo)
 		if (VGZ_ObufStorage(bo, vg))
 			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-		bo->fetch_obj->len += dl;
+		VFP_update_length(bo, dl);
 	} while (i != Z_STREAM_END);
 	VGZ_UpdateObj(vg, bo->fetch_obj);
 	if (VGZ_Destroy(&vg) != VGZ_END)
@@ -627,7 +627,7 @@ vfp_testgzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
 		bytes -= wl;
 		VGZ_Ibuf(vg, st->ptr + st->len, wl);
 		st->len += wl;
-		bo->fetch_obj->len += wl;
+		VFP_update_length(bo, wl);
 
 		while (!VGZ_IbufEmpty(vg)) {
 			VGZ_Obuf(vg, vg->m_buf, vg->m_sz);



From perbu at varnish-cache.org  Thu Mar 29 09:13:35 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Thu, 29 Mar 2012 11:13:35 +0200
Subject: [master] 70468df add a label for refs
Message-ID: <E1SDBQF-0001ii-8E@project.varnish-software.com>

commit 70468df3126778832547f1b9085be23010c00a7b
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 29 11:06:30 2012 +0200

    add a label for refs

diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst
index 54bce37..da9a340 100644
--- a/doc/sphinx/tutorial/vcl.rst
+++ b/doc/sphinx/tutorial/vcl.rst
@@ -49,6 +49,8 @@ In vcl_fetch you still have the request object, req, available. There
 is also a *backend response*, beresp. beresp will contain the HTTP
 headers from the backend.
 
+.. _tutorial-vcl_fetch_actions:
+
 
 actions
 ~~~~~~~



From perbu at varnish-cache.org  Thu Mar 29 09:13:35 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Thu, 29 Mar 2012 11:13:35 +0200
Subject: [master] 56cfc90 alternative, simpler cookie stripper by James Light
Message-ID: <E1SDBQF-0001in-9F@project.varnish-software.com>

commit 56cfc9088bd7dd888b0d13517193d9ba043219b9
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 29 11:10:36 2012 +0200

    alternative, simpler cookie stripper by James Light

diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst
index 20fd302..93289c3 100644
--- a/doc/sphinx/tutorial/cookies.rst
+++ b/doc/sphinx/tutorial/cookies.rst
@@ -64,8 +64,25 @@ cookies named COOKIE1 and COOKIE2 and you can marvel at it::
       }
   }
 
-The example is taken from the Varnish Wiki, where you can find other
-scary examples of what can be done in VCL.
+A somewhat simpler example that can accomplish almost the same can be
+found below. Instead of filtering out the other cookies it picks out
+the one cookie that is needed, copies it to another header and then
+copies it back, deleting the original cookie header.::
+
+  sub vcl_recv {
+         # save the original cookie header so we can mangle it
+        set req.http.X-Varnish-PHP_SID = req.http.Cookie;
+        # using a capturing sub pattern, extract the continuous string of 
+        # alphanumerics that immediately follows "PHPSESSID="
+        set req.http.X-Varnish-PHP_SID = 
+           regsuball(req.http.X-Varnish-PHP_SID, ";? ?PHPSESSID=([a-zA-Z0-9]+)( |;| ;).*","\1");
+        set req.http.Cookie = req.X-Varnish-PHP_SID;
+        remove req.X-Varnish-PHP_SID;
+   }   
+
+There are other scary examples of what can be done in VCL in the
+Varnish Cache Wiki.
+
 
 Cookies coming from the backend
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~



From perbu at varnish-cache.org  Thu Mar 29 09:46:09 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Thu, 29 Mar 2012 11:46:09 +0200
Subject: [master] f93c6fa kill warnings. fix a ref
Message-ID: <E1SDBvl-0006eY-7D@project.varnish-software.com>

commit f93c6faae2817ef4e55f96359da2ff44d77f30f9
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Mar 29 11:46:04 2012 +0200

    kill warnings. fix a ref

diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst
index 93289c3..88e9d07 100644
--- a/doc/sphinx/tutorial/cookies.rst
+++ b/doc/sphinx/tutorial/cookies.rst
@@ -89,6 +89,6 @@ Cookies coming from the backend
 
 If your backend server sets a cookie using the Set-Cookie header
 Varnish will not cache the page.  A hit-for-pass object (see
-:ref:`tutorial_vcl_fetch_actions`) is created. So, if the backend
+:ref:`tutorial-vcl_fetch_actions`) is created. So, if the backend
 server acts silly and sets unwanted cookies just unset the Set-Cookie
 header and all should be fine. 
diff --git a/doc/sphinx/tutorial/devicedetection.rst b/doc/sphinx/tutorial/devicedetection.rst
index 9f02493..6bfc4c9 100644
--- a/doc/sphinx/tutorial/devicedetection.rst
+++ b/doc/sphinx/tutorial/devicedetection.rst
@@ -61,6 +61,7 @@ is dependant on this header.
 Everything works out of the box from Varnish' perspective.
 
 .. 071-example1-start
+
 VCL::
 
     sub vcl_recv { 
@@ -95,6 +96,7 @@ VCL::
             set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent");
         }
     }
+
 .. 071-example1-end
 
 Example 2: Normalize the User-Agent string
@@ -118,6 +120,7 @@ A possible use for this is for CGI scripts where only a small set of predefined
 headers are (by default) available for the script.
 
 .. 072-example2-start
+
 VCL::
 
     sub vcl_recv { 
@@ -158,6 +161,7 @@ The client itself does not see this classification, only the backend request
 is changed.
 
 .. 073-example3-start
+
 VCL::
 
     sub vcl_recv { 
@@ -240,6 +244,7 @@ Redirecting mobile clients
 If you want to redirect mobile clients you can use the following snippet.
 
 .. 065-redir-mobile-start
+
 VCL::
 
     sub vcl_recv {
diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst
index da9a340..0601468 100644
--- a/doc/sphinx/tutorial/vcl.rst
+++ b/doc/sphinx/tutorial/vcl.rst
@@ -51,7 +51,6 @@ headers from the backend.
 
 .. _tutorial-vcl_fetch_actions:
 
-
 actions
 ~~~~~~~