From phk at FreeBSD.org  Mon Jun  1 07:29:34 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 01 Jun 2015 09:29:34 +0200
Subject: [master] 321b3b4 Make sure all "table" .h files, and only those,
 have FlexeLint shutuppery about indentation.
Message-ID: <E1YzKAA-00046s-RC@project.varnish-software.com>


commit 321b3b460d4ac7ca65c3686c6e55eece525a2c07
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 1 07:13:08 2015 +0000

    Make sure all "table" .h files, and only those, have FlexeLint
    shutuppery about indentation.

diff --git a/include/tbl/acct_fields_req.h b/include/tbl/acct_fields_req.h
index a1759f5..fece3dc 100644
--- a/include/tbl/acct_fields_req.h
+++ b/include/tbl/acct_fields_req.h
@@ -30,7 +30,9 @@
  * NB: Remember to mark those in vsc_fields.h to be included in struct dstat.
  */
 
+/*lint -save -e525 -e539 */
 ACCT(req_hdrbytes)
 ACCT(req_bodybytes)
 ACCT(resp_hdrbytes)
 ACCT(resp_bodybytes)
+/*lint -restore */
diff --git a/include/tbl/backend_poll.h b/include/tbl/backend_poll.h
index a9ee3a3..04174cb 100644
--- a/include/tbl/backend_poll.h
+++ b/include/tbl/backend_poll.h
@@ -27,6 +27,7 @@
  *
  */
 
+/*lint -save -e525 -e539 */
 BITMAP(good_ipv4, '4', "Good IPv4", 0)
 BITMAP(good_ipv6, '6', "Good IPv6", 0)
 BITMAP( err_xmit, 'x', "Error Xmit", 0)
@@ -34,3 +35,4 @@ BITMAP(good_xmit, 'X', "Good Xmit", 0)
 BITMAP( err_recv, 'r', "Error Recv", 0)
 BITMAP(good_recv, 'R', "Good Recv", 0)
 BITMAP(happy,     'H', "Happy", 1)
+/* lint -restore */
diff --git a/include/tbl/ban_vars.h b/include/tbl/ban_vars.h
index a3df78b..7bc665d 100644
--- a/include/tbl/ban_vars.h
+++ b/include/tbl/ban_vars.h
@@ -29,7 +29,9 @@
  *
  */
 
+/*lint -save -e525 -e539 */
 PVAR("req.url",		BANS_FLAG_REQ,			BANS_ARG_URL)
 PVAR("req.http.",	BANS_FLAG_REQ | BANS_FLAG_HTTP,	BANS_ARG_REQHTTP)
 PVAR("obj.status",	BANS_FLAG_OBJ,			BANS_ARG_OBJSTATUS)
 PVAR("obj.http.",	BANS_FLAG_OBJ | BANS_FLAG_HTTP,	BANS_ARG_OBJHTTP)
+/*lint -restore */
diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h
index ba9400a..31eef0c 100644
--- a/include/tbl/debug_bits.h
+++ b/include/tbl/debug_bits.h
@@ -29,6 +29,7 @@
  *
  */
 
+/*lint -save -e525 -e539 */
 DEBUG_BIT(REQ_STATE,		req_state,	"VSL Request state engine")
 DEBUG_BIT(WORKSPACE,		workspace,	"VSL Workspace operations")
 DEBUG_BIT(WAITER,		waiter,		"VSL Waiter internals")
@@ -40,3 +41,4 @@ DEBUG_BIT(LURKER,		lurker,		"VSL Ban lurker")
 DEBUG_BIT(ESI_CHOP,		esi_chop,	"Chop ESI fetch to bits")
 DEBUG_BIT(FLUSH_HEAD,		flush_head,	"Flush after http1 head")
 DEBUG_BIT(VTC_MODE,		vtc_mode,	"Varnishtest Mode")
+/*lint -restore */
diff --git a/include/tbl/feature_bits.h b/include/tbl/feature_bits.h
index 388f9da..6fcf279 100644
--- a/include/tbl/feature_bits.h
+++ b/include/tbl/feature_bits.h
@@ -29,6 +29,7 @@
  *
  */
 
+/*lint -save -e525 -e539 */
 FEATURE_BIT(SHORT_PANIC,		short_panic,
     "Short panic message.",
     "Reduce level of detail for panic messages."
@@ -58,3 +59,4 @@ FEATURE_BIT(ESI_REMOVE_BOM,		esi_remove_bom,
     "Remove UTF-8 BOM from front of object."
     "Ignore and remove the UTF-8 BOM (0xeb 0xbb 0xbf) from front of object."
 )
+/*lint -restore */
diff --git a/include/tbl/http_response.h b/include/tbl/http_response.h
index f60820c..6baacfb 100644
--- a/include/tbl/http_response.h
+++ b/include/tbl/http_response.h
@@ -28,6 +28,7 @@
  *
  */
 
+/*lint -save -e525 -e539 */
 HTTP_RESP(101, "Switching Protocols")
 HTTP_RESP(200, "OK")
 HTTP_RESP(201, "Created")
@@ -68,3 +69,4 @@ HTTP_RESP(502, "Bad Gateway")
 HTTP_RESP(503, "Service Unavailable")
 HTTP_RESP(504, "Gateway Timeout")
 HTTP_RESP(505, "HTTP Version Not Supported")
+/*lint -restore */
diff --git a/include/tbl/vsc_all.h b/include/tbl/vsc_all.h
index 6cc1805..0ef0b08 100644
--- a/include/tbl/vsc_all.h
+++ b/include/tbl/vsc_all.h
@@ -28,6 +28,7 @@
  */
 
 
+/*lint -save -e525 -e539 */
 VSC_DO(MAIN, main, VSC_type_main)
 #include "tbl/vsc_f_main.h"
 VSC_DONE(MAIN, main, VSC_type_main)
@@ -67,3 +68,4 @@ VSC_DO(LCK, lck, VSC_type_lck)
 #include "tbl/vsc_fields.h"
 #undef VSC_DO_LCK
 VSC_DONE(LCK, lck, VSC_type_lck)
+/*lint -restore */
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index ecb2683..fff29d5 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -42,6 +42,7 @@
  *	Long Description (in RST "definition list" format)
  */
 
+/*lint -save -e525 -e539 */
 #define NODEF_NOTICE \
     "NB: This log record is masked by default.\n\n"
 
@@ -518,3 +519,4 @@ SLTM(VfpAcct, 0, "Fetch filter accounting",
 )
 
 #undef NODEF_NOTICE
+/*lint -restore */
diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h
index ab09d70..82a2219 100644
--- a/include/tbl/vsl_tags_http.h
+++ b/include/tbl/vsl_tags_http.h
@@ -43,6 +43,7 @@
  *
  */
 
+/*lint -save -e525 -e539 */
 SLTH(Method,	HTTP_HDR_METHOD,	1, 0, "method",
 	"The HTTP request method used.\n\n"
 )
@@ -79,3 +80,4 @@ SLTH(Unset,	HTTP_HDR_UNSET,		0, 0, "unset header",
 SLTH(Lost,	HTTP_HDR_LOST,		0, 0, "lost header",
 	""
 )
+/*lint -restore */
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 2ec667c..7ea599d 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -432,7 +432,6 @@ vsc_build_pt_list(struct VSM_data *vd)
 	vsc_delete_pt_list(vsc);
 
 	VTAILQ_FOREACH(vf, &vsc->vf_list, list) {
-		/*lint -save -e525 -e539 */
 #define VSC_DO(U,l,t)						\
 		CHECK_OBJ_NOTNULL(vf, VSC_VF_MAGIC);		\
 		if (!strcmp(vf->fantom.type, t))		\
@@ -443,7 +442,6 @@ vsc_build_pt_list(struct VSM_data *vd)
 #undef VSC_DO
 #undef VSC_F
 #undef VSC_DONE
-		/*lint -restore */
 	}
 }
 



From phk at FreeBSD.org  Mon Jun  1 07:29:34 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 01 Jun 2015 09:29:34 +0200
Subject: [master] ff0b761 Always reset backend poll to initial state when VCL
 changes temperature.
Message-ID: <E1YzKAA-00046z-SP@project.varnish-software.com>


commit ff0b76118416c912af88f69a3a2a940edcd9655c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 1 07:29:10 2015 +0000

    Always reset backend poll to initial state when VCL changes temperature.

diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index b6a0235..641ed4e 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -91,6 +91,112 @@ static pthread_cond_t			vbp_cond;
 static struct binheap			*vbp_heap;
 
 /*--------------------------------------------------------------------
+ * Record pokings...
+ */
+
+static void
+vbp_start_poke(struct vbp_target *vt)
+{
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
+#define BITMAP(n, c, t, b) \
+	vt->n <<= 1;
+#include "tbl/backend_poll.h"
+#undef BITMAP
+
+	vt->last = 0;
+	vt->resp_buf[0] = '\0';
+}
+
+static void
+vbp_has_poked(struct vbp_target *vt)
+{
+	unsigned i, j;
+	uint64_t u;
+
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
+	/* Calculate exponential average */
+	if (vt->happy & 1) {
+		if (vt->rate < AVG_RATE)
+			vt->rate += 1.0;
+		vt->avg += (vt->last - vt->avg) / vt->rate;
+	}
+
+	u = vt->happy;
+	for (i = j = 0; i < vt->probe.window; i++) {
+		if (u & 1)
+			j++;
+		u >>= 1;
+	}
+	vt->good = j;
+}
+
+static void
+vbp_update_backend(struct vbp_target *vt)
+{
+	unsigned i;
+	char bits[10];
+	const char *logmsg;
+
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
+	Lck_Lock(&vbp_mtx);
+	if (vt->backend != NULL) {
+		i = 0;
+#define BITMAP(n, c, t, b) \
+		bits[i++] = (vt->n & 1) ? c : '-';
+#include "tbl/backend_poll.h"
+#undef BITMAP
+		bits[i] = '\0';
+
+		if (vt->good >= vt->probe.threshold) {
+			if (vt->backend->healthy)
+				logmsg = "Still healthy";
+			else {
+				logmsg = "Back healthy";
+				vt->backend->health_changed = VTIM_real();
+			}
+			vt->backend->healthy = 1;
+		} else {
+			if (vt->backend->healthy) {
+				logmsg = "Went sick";
+				vt->backend->health_changed = VTIM_real();
+			} else
+				logmsg = "Still sick";
+			vt->backend->healthy = 0;
+		}
+		VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s",
+		    vt->backend->display_name, logmsg, bits,
+		    vt->good, vt->probe.threshold, vt->probe.window,
+		    vt->last, vt->avg, vt->resp_buf);
+		if (vt->backend != NULL && vt->backend->vsc != NULL)
+			vt->backend->vsc->happy = vt->happy;
+	}
+	Lck_Unlock(&vbp_mtx);
+}
+
+static void
+vbp_reset(struct vbp_target *vt)
+{
+	unsigned u;
+
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+	vt->avg = 0.0;
+	vt->rate = 0.0;
+#define BITMAP(n, c, t, b) \
+	vt->n = 0;
+#include "tbl/backend_poll.h"
+#undef BITMAP
+
+	for (u = 0; u < vt->probe.initial; u++) {
+		vbp_start_poke(vt);
+		vt->happy |= 1;
+		vbp_has_poked(vt);
+	}
+}
+
+/*--------------------------------------------------------------------
  * Poke one backend, once, but possibly at both IPv4 and IPv6 addresses.
  *
  * We do deliberately not use the stuff in cache_backend.c, because we
@@ -193,82 +299,6 @@ vbp_poke(struct vbp_target *vt)
 }
 
 /*--------------------------------------------------------------------
- * Record pokings...
- */
-
-static void
-vbp_start_poke(struct vbp_target *vt)
-{
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-
-#define BITMAP(n, c, t, b)	vt->n <<= 1;
-#include "tbl/backend_poll.h"
-#undef BITMAP
-
-	vt->last = 0;
-	vt->resp_buf[0] = '\0';
-}
-
-static void
-vbp_has_poked(struct vbp_target *vt)
-{
-	unsigned i, j;
-	uint64_t u;
-	const char *logmsg;
-	char bits[10];
-
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-
-	/* Calculate exponential average */
-	if (vt->happy & 1) {
-		if (vt->rate < AVG_RATE)
-			vt->rate += 1.0;
-		vt->avg += (vt->last - vt->avg) / vt->rate;
-	}
-
-	i = 0;
-#define BITMAP(n, c, t, b)	bits[i++] = (vt->n & 1) ? c : '-';
-#include "tbl/backend_poll.h"
-#undef BITMAP
-	bits[i] = '\0';
-
-	u = vt->happy;
-	for (i = j = 0; i < vt->probe.window; i++) {
-		if (u & 1)
-			j++;
-		u >>= 1;
-	}
-	vt->good = j;
-
-	Lck_Lock(&vbp_mtx);
-	if (vt->backend != NULL) {
-		if (vt->good >= vt->probe.threshold) {
-			if (vt->backend->healthy)
-				logmsg = "Still healthy";
-			else {
-				logmsg = "Back healthy";
-				vt->backend->health_changed = VTIM_real();
-			}
-			vt->backend->healthy = 1;
-		} else {
-			if (vt->backend->healthy) {
-				logmsg = "Went sick";
-				vt->backend->health_changed = VTIM_real();
-			} else
-				logmsg = "Still sick";
-			vt->backend->healthy = 0;
-		}
-		VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s",
-		    vt->backend->display_name, logmsg, bits,
-		    vt->good, vt->probe.threshold, vt->probe.window,
-		    vt->last, vt->avg, vt->resp_buf);
-		if (vt->backend != NULL && vt->backend->vsc != NULL)
-			vt->backend->vsc->happy = vt->happy;
-	}
-	Lck_Unlock(&vbp_mtx);
-}
-
-/*--------------------------------------------------------------------
  */
 
 static void __match_proto__(task_func_t)
@@ -285,6 +315,7 @@ vbp_task(struct worker *wrk, void *priv)
 	vbp_start_poke(vt);
 	vbp_poke(vt);
 	vbp_has_poked(vt);
+	vbp_update_backend(vt);
 
 	Lck_Lock(&vbp_mtx);
 	if (vt->running < 0) {
@@ -296,6 +327,7 @@ vbp_task(struct worker *wrk, void *priv)
 	}
 	Lck_Unlock(&vbp_mtx);
 }
+
 /*--------------------------------------------------------------------
  */
 
@@ -469,7 +501,9 @@ VBP_Control(const struct backend *be, int enable)
 	vt = be->probe;
 	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
 
-VSL(SLT_Debug, 0, "VBP_CONTROL %d", enable);
+	vbp_reset(vt);
+	vbp_update_backend(vt);
+
 	Lck_Lock(&vbp_mtx);
 	if (enable) {
 		assert(vt->heap_idx == BINHEAP_NOIDX);
@@ -492,7 +526,6 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *p,
     const char *hosthdr)
 {
 	struct vbp_target *vt;
-	unsigned u;
 
 	ASSERT_CLI();
 	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
@@ -507,20 +540,15 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *p,
 	AN(vt->tcp_pool);
 
 	vt->probe = *p;
-	vt->backend = b;
-
 	vbp_set_defaults(vt);
+
 	vbp_build_req(vt, hosthdr);
 
-	for (u = 0; u < vt->probe.initial; u++) {
-		if (u)
-			vbp_has_poked(vt);
-		vbp_start_poke(vt);
-		vt->happy |= 1;
-		vbp_has_poked(vt);
-	}
+	vt->backend = b;
 	b->probe = vt;
-	vbp_has_poked(vt);
+
+	vbp_reset(vt);
+	vbp_update_backend(vt);
 }
 
 void
@@ -548,8 +576,8 @@ VBP_Remove(struct backend *be)
 		FREE_OBJ(vt);
 	}
 }
-/*--------------------------------------------------------------------
- */
+
+/*-------------------------------------------------------------------*/
 
 static int __match_proto__(binheap_cmp_t)
 vbp_cmp(void *priv, const void *a, const void *b)
@@ -573,8 +601,7 @@ vbp_update(void *priv, void *p, unsigned u)
 	vt->heap_idx = u;
 }
 
-/*--------------------------------------------------------------------
- */
+/*-------------------------------------------------------------------*/
 
 void
 VBP_Init(void)



From phk at FreeBSD.org  Mon Jun  1 09:33:24 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 01 Jun 2015 11:33:24 +0200
Subject: [master] 256504c Comment polishing
Message-ID: <E1YzM60-00080M-MO@project.varnish-software.com>


commit 256504c5a658a3a6409742cb931104c2e1d54e7c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 1 08:10:02 2015 +0000

    Comment polishing

diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index b917618..9fa64b5 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -73,9 +73,7 @@ VBE_DeleteBackend(struct backend *b)
 }
 
 /*--------------------------------------------------------------------
- * Add a backend/director instance when loading a VCL.
- * If an existing backend is matched, grab a refcount and return.
- * Else create a new backend structure with reference initialized to one.
+ * Create a new director::backend instance.
  */
 
 struct backend *
@@ -102,9 +100,6 @@ VBE_AddBackend(const char *vcl, const struct vrt_backend *vb)
 
 	b->tcp_pool = VBT_Ref(vb->ipv4_suckaddr, vb->ipv6_suckaddr);
 
-	/*
-	 * Copy over the sockaddrs
-	 */
 	if (vb->ipv4_suckaddr != NULL)
 		b->ipv4 = VSA_Clone(vb->ipv4_suckaddr);
 	if (vb->ipv6_suckaddr != NULL)



From phk at FreeBSD.org  Mon Jun  1 09:33:24 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 01 Jun 2015 11:33:24 +0200
Subject: [master] 0aa8782 Get rid of the array of backend/directors in the
 compiled VCL.
Message-ID: <E1YzM60-00080S-Nj@project.varnish-software.com>


commit 0aa8782e582cb84284845975a340497f716dece8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 1 09:32:59 2015 +0000

    Get rid of the array of backend/directors in the compiled VCL.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 653a0cd..c4d3ab4 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -649,7 +649,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 
 	/* By default we use the first backend */
 	AZ(req->director_hint);
-	req->director_hint = req->vcl->director[0];
+	req->director_hint = *req->vcl->default_director;
 	AN(req->director_hint);
 
 	req->d_ttl = -1;
diff --git a/bin/varnishtest/tests/r00916.vtc b/bin/varnishtest/tests/r00916.vtc
index 04aa7c6..8057f1a 100644
--- a/bin/varnishtest/tests/r00916.vtc
+++ b/bin/varnishtest/tests/r00916.vtc
@@ -5,7 +5,7 @@ server s1 {
 	txresp -body "FOO"
 } -start
 
-varnish v1 -errvcl {Symbol not found: 's-1' (expected type BACKEND)} {
+varnish v1 -errvcl {Backend not found: 's-1'} {
 	backend b { .host = "127.0.0.1"; }
 	sub s1 {
 	}
diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 3198e89..990f39e 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -118,3 +118,12 @@ varnish v1 -errvcl {'bereq.between_bytes_timeout': cannot be set} {
 		set bereq.between_bytes_timeout = 10s;
 	}
 }
+
+varnish v1 -errvcl {Backend not found: 'c'} {
+	backend b { .host = "127.0.0.1"; }
+	sub vcl_backend_response {
+		if (beresp.backend == c) {
+			set beresp.ttl = 1h;
+		}
+	}
+}
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index f695dc7..54560bf 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -1060,8 +1060,7 @@ struct VCL_conf {
 
 	char		*loaded_name;
 
-	struct director	**director;
-	unsigned	ndirector;
+	struct director	**default_director;
 	struct vrt_ref	*ref;
 	unsigned	nref;
 	unsigned	busy;
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index b937404..d788d28 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -262,7 +262,7 @@ vcc_ParseProbe(struct vcc *tl)
  */
 
 static void
-vcc_ParseHostDef(struct vcc *tl, const struct token *t_be)
+vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 {
 	struct token *t_field;
 	struct token *t_host = NULL;
@@ -273,11 +273,6 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be)
 	struct vsb *vsb;
 	unsigned u;
 	double t;
-	char vgcname[MAX_BACKEND_NAME + 8];
-
-	sprintf(vgcname, "_%.*s", PF(t_be));
-
-	Fh(tl, 1, "\n#define VGC_backend_%s %d\n", vgcname, tl->ndirector);
 
 	fs = vcc_FldSpec(tl,
 	    "!host",
@@ -415,15 +410,14 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be)
 
 	ifp = New_IniFin(tl);
 	VSB_printf(ifp->ini,
-	    "\tVRT_init_vbe(ctx, &VGCDIR(%s), &vgc_dir_priv_%s);",
+	    "\tVRT_init_vbe(ctx, &%s, &vgc_dir_priv_%s);",
 	    vgcname, vgcname);
 	VSB_printf(ifp->fin,
-	    "\tVRT_fini_vbe(ctx, &VGCDIR(%s), &vgc_dir_priv_%s);",
+	    "\tVRT_fini_vbe(ctx, &%s, &vgc_dir_priv_%s);",
 	    vgcname, vgcname);
 	VSB_printf(ifp->event,
-	    "\tVRT_event_vbe(ctx, ev, VGCDIR(%s), &vgc_dir_priv_%s);",
+	    "\tVRT_event_vbe(ctx, ev, %s, &vgc_dir_priv_%s);",
 	    vgcname, vgcname);
-	tl->ndirector++;
 }
 
 /*--------------------------------------------------------------------
@@ -434,8 +428,8 @@ void
 vcc_ParseBackend(struct vcc *tl)
 {
 	struct token *t_first, *t_be;
-	int isfirst;
 	struct symbol *sym;
+	char vgcname[MAX_BACKEND_NAME + 20];
 
 	t_first = tl->t;
 	vcc_NextToken(tl);		/* ID: backend */
@@ -455,7 +449,8 @@ vcc_ParseBackend(struct vcc *tl)
 	t_be = tl->t;
 	vcc_NextToken(tl);
 
-	isfirst = tl->ndirector;
+	sprintf(vgcname, "vgc_backend_%.*s", PF(t_be));
+	Fh(tl, 0, "static struct director *%s;\n", vgcname);
 
 	sym = VCC_GetSymbolTok(tl, t_be, SYM_BACKEND);
 	AN(sym);
@@ -466,10 +461,11 @@ vcc_ParseBackend(struct vcc *tl)
 	}
 	sym->fmt = BACKEND;
 	sym->eval = vcc_Eval_Backend;
+	sym->eval_priv = TlDup(tl, vgcname);
 	sym->ndef++;
 	ERRCHK(tl);
 
-	vcc_ParseHostDef(tl, t_be);
+	vcc_ParseHostDef(tl, t_be, vgcname);
 	ERRCHK(tl);
 
 	if (tl->err) {
@@ -479,8 +475,8 @@ vcc_ParseBackend(struct vcc *tl)
 		return;
 	}
 
-	if (isfirst == 1 || vcc_IdIs(t_be, "default")) {
-		tl->defaultdir = tl->ndirector - 1;
-		tl->t_defaultdir = t_be;
+	if (tl->default_director == NULL || vcc_IdIs(t_be, "default")) {
+		tl->default_director = sym->eval_priv;
+		tl->t_default_director = t_be;
 	}
 }
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 4717148..06f3a3e 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -387,14 +387,10 @@ EmitStruct(const struct vcc *tl)
 	}
 	Fc(tl, 0, "};\n");
 
-	Fc(tl, 0, "\nstatic struct director\t*directors[%d];\n",
-	    tl->ndirector);
-
 	Fc(tl, 0, "\nconst struct VCL_conf VCL_conf = {\n");
 	Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n");
 	Fc(tl, 0, "\t.event_vcl = VGC_Event,\n");
-	Fc(tl, 0, "\t.ndirector = %d,\n", tl->ndirector);
-	Fc(tl, 0, "\t.director = directors,\n");
+	Fc(tl, 0, "\t.default_director = &%s,\n", tl->default_director);
 	Fc(tl, 0, "\t.ref = VGC_ref,\n");
 	Fc(tl, 0, "\t.nref = VGC_NREFS,\n");
 	Fc(tl, 0, "\t.nsrc = %u,\n", tl->nsources);
@@ -535,7 +531,6 @@ vcc_NewVcc(const struct vcc *tl0)
 	VTAILQ_INIT(&tl->sources);
 
 	tl->nsources = 0;
-	tl->ndirector = 1;
 
 	/* General C code */
 	tl->fc = VSB_new_auto();
@@ -600,7 +595,6 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	struct vcc *tl;
 	struct symbol *sym;
 	const struct var *v;
-	struct inifin *ifp;
 	char *of;
 	int i;
 
@@ -629,9 +623,6 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	Fh(tl, 0, "/* ---===### VCC generated code ###===---*/\n");
 	Fh(tl, 0, "\nextern const struct VCL_conf VCL_conf;\n");
 
-	/* Macro for accessing directors */
-	Fh(tl, 0, "#define VGCDIR(n) VCL_conf.director[VGC_backend_##n]\n");
-
 	/* Register and lex the main source */
 	VTAILQ_INSERT_TAIL(&tl->sources, sp, list);
 	sp->idx = tl->nsources++;
@@ -665,7 +656,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 		return (vcc_DestroyTokenList(tl, NULL));
 
 	/* Check if we have any backends at all */
-	if (tl->ndirector == 1) {
+	if (tl->default_director == NULL) {
 		VSB_printf(tl->sb,
 		    "No backends or directors found in VCL program, "
 		    "at least one is necessary.\n");
@@ -674,11 +665,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	}
 
 	/* Configure the default director */
-	ifp = New_IniFin(tl);
-	VSB_printf(ifp->ini,
-	    "\tVCL_conf.director[0] = VCL_conf.director[%d];",
-	    tl->defaultdir);
-	vcc_AddRef(tl, tl->t_defaultdir, SYM_BACKEND);
+	vcc_AddRef(tl, tl->t_default_director, SYM_BACKEND);
 
 	/* Check for orphans */
 	if (vcc_CheckReferences(tl))
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index b2e15e3..a222b9b 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -189,7 +189,6 @@ struct vcc {
 	struct vsb		*fm[VCL_MET_MAX];	/* Method bodies */
 	struct vsb		*sb;
 	int			err;
-	int			ndirector;
 	struct proc		*curproc;
 	struct proc		*mprocs[VCL_MET_MAX];
 
@@ -197,8 +196,8 @@ struct vcc {
 
 	int			nprobe;
 
-	int			defaultdir;
-	struct token		*t_defaultdir;
+	const char		*default_director;
+	struct token		*t_default_director;
 
 	unsigned		unique;
 
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 6a03bff..d3ad332 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -508,7 +508,7 @@ vcc_Eval_Backend(struct vcc *tl, struct expr **e, const struct symbol *sym)
 
 	vcc_ExpectCid(tl);
 	vcc_AddRef(tl, tl->t, SYM_BACKEND);
-	*e = vcc_mk_expr(BACKEND, "VGCDIR(_%.*s)", PF(tl->t));
+	*e = vcc_mk_expr(BACKEND, "%s", sym->eval_priv);
 	(*e)->constant = EXPR_VAR;	/* XXX ? */
 	vcc_NextToken(tl);
 }
@@ -1129,6 +1129,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt)
 	const char *re;
 	const char *not;
 	struct token *tk;
+	struct symbol *sym;
 
 	*e = NULL;
 
@@ -1188,10 +1189,19 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt)
 	}
 	if ((*e)->fmt == BACKEND &&
 	    (tl->t->tok == T_EQ || tl->t->tok == T_NEQ)) {
+		// XXX: just ask for a BACKEND expression instead ?
 		vcc_NextToken(tl);
 		ExpectErr(tl, ID);
+		sym = VCC_FindSymbol(tl, tl->t, SYM_BACKEND);
+		if (sym == NULL) {
+			VSB_printf(tl->sb, "Backend not found: ");
+			vcc_ErrToken(tl, tl->t);
+			VSB_printf(tl->sb, "\n");
+			vcc_ErrWhere(tl, tl->t);
+			return;
+		}
 		vcc_AddRef(tl, tl->t, SYM_BACKEND);
-		bprintf(buf, "(\v1 %.*s VGCDIR(_%.*s))", PF(tk), PF(tl->t));
+		bprintf(buf, "(\v1 %.*s %s)", PF(tk), sym->eval_priv);
 		vcc_NextToken(tl);
 		*e = vcc_expr_edit(BOOL, buf, *e, NULL);
 		return;



From phk at FreeBSD.org  Mon Jun  1 09:55:41 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 01 Jun 2015 11:55:41 +0200
Subject: [master] e62cf00 Add casts to make gcc happier.
Message-ID: <E1YzMRZ-0002GY-Ud@project.varnish-software.com>


commit e62cf00f7b3de01cba4cfb369a0b06bbaee4df68
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 1 09:55:30 2015 +0000

    Add casts to make gcc happier.

diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index d3ad332..df5aa43 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -508,7 +508,7 @@ vcc_Eval_Backend(struct vcc *tl, struct expr **e, const struct symbol *sym)
 
 	vcc_ExpectCid(tl);
 	vcc_AddRef(tl, tl->t, SYM_BACKEND);
-	*e = vcc_mk_expr(BACKEND, "%s", sym->eval_priv);
+	*e = vcc_mk_expr(BACKEND, "%s", (const char *)sym->eval_priv);
 	(*e)->constant = EXPR_VAR;	/* XXX ? */
 	vcc_NextToken(tl);
 }
@@ -1201,7 +1201,8 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt)
 			return;
 		}
 		vcc_AddRef(tl, tl->t, SYM_BACKEND);
-		bprintf(buf, "(\v1 %.*s %s)", PF(tk), sym->eval_priv);
+		bprintf(buf, "(\v1 %.*s %s)", PF(tk),
+		    (const char *)sym->eval_priv);
 		vcc_NextToken(tl);
 		*e = vcc_expr_edit(BOOL, buf, *e, NULL);
 		return;



From nils.goroll at uplex.de  Mon Jun  1 11:54:17 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 01 Jun 2015 13:54:17 +0200
Subject: [master] a72ab6c We don't have a vsl when coming from error in
 HTTP1_Session()
Message-ID: <E1YzOIL-0006ss-GI@project.varnish-software.com>


commit a72ab6cf297b97b25d7a392ef0202b8cfec8a6a0
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jun 1 13:50:04 2015 +0200

    We don't have a vsl when coming from error in HTTP1_Session()
    
    Fixes #1745

diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 4f02e26..f4e4f61 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -167,11 +167,12 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	VRTPRIV_dynamic_kill(sp->privs, (uintptr_t)&req->top);
 
 	/* Charge and log byte counters */
-	AN(req->vsl->wid);
-	CNT_AcctLogCharge(wrk->stats, req);
+	if (req->vsl->wid) {
+		CNT_AcctLogCharge(wrk->stats, req);
+		VSL_End(req->vsl);
+	}
 	req->req_bodybytes = 0;
 
-	VSL_End(req->vsl);
 
 	if (!isnan(req->t_prev) && req->t_prev > 0.)
 		sp->t_idle = req->t_prev;



From martin at varnish-software.com  Mon Jun  1 12:21:37 2015
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 01 Jun 2015 14:21:37 +0200
Subject: [master] 977b831 Update the users guide to for new -sfile syntax
Message-ID: <E1YzOin-0001OB-Ab@project.varnish-software.com>


commit 977b831d7c21b392104a4156e4ea5af1d9d68f65
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 1 14:19:05 2015 +0200

    Update the users guide to for new -sfile syntax
    
    It used to be possible to specify -sfile size in percentage. This
    feature has been removed. Update the users guide to reflect this.
    
    Fixes: #1744

diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst
index 6664d8d..3539174 100644
--- a/doc/sphinx/users-guide/storage-backends.rst
+++ b/doc/sphinx/users-guide/storage-backends.rst
@@ -69,10 +69,9 @@ suffixes:
 
       T, t    The size is expressed in tebibytes.
 
-      %       The size is expressed as a percentage of the free space on the
-              file system where it resides.
-
-The default size is to use 50% of the space available on the device.
+If 'path' points to an existing file and no size is specified, the
+size of the existing file will be used. If 'path' does not point to an
+existing file it is an error to not specify the size.
 
 If the backing file already exists, it will be truncated or expanded
 to the specified size.



From martin at varnish-software.com  Mon Jun  1 12:23:32 2015
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 01 Jun 2015 14:23:32 +0200
Subject: [master] 900e4de Another mention of % in -sfile docs
Message-ID: <E1YzOke-0003rS-Rw@project.varnish-software.com>


commit 900e4def9a7d93c2498ee2748417d652508039e0
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 1 14:23:09 2015 +0200

    Another mention of % in -sfile docs

diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst
index 3539174..5d08d1e 100644
--- a/doc/sphinx/users-guide/storage-backends.rst
+++ b/doc/sphinx/users-guide/storage-backends.rst
@@ -86,7 +86,7 @@ the storage file using `dd(1)` will reduce fragmentation to a minimum.
 The 'granularity' parameter specifies the granularity of
 allocation. All allocations are rounded up to this size. The granularity is
 is assumed to be expressed in bytes, unless followed by one of the
-suffixes described for size except for %.
+suffixes described for size.
 
 The default granularity is the VM page size. The size should be reduced if you
 have many small objects.



From martin at varnish-software.com  Mon Jun  1 12:25:06 2015
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 01 Jun 2015 14:25:06 +0200
Subject: [4.0] 57d2e7a Update the users guide to for new -sfile syntax
Message-ID: <E1YzOmA-0006RL-1c@project.varnish-software.com>


commit 57d2e7a8dedd214fee2ecae436bcfe4d67d228e9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 1 14:19:05 2015 +0200

    Update the users guide to for new -sfile syntax
    
    It used to be possible to specify -sfile size in percentage. This
    feature has been removed. Update the users guide to reflect this.
    
    Fixes: #1744

diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst
index 6664d8d..3539174 100644
--- a/doc/sphinx/users-guide/storage-backends.rst
+++ b/doc/sphinx/users-guide/storage-backends.rst
@@ -69,10 +69,9 @@ suffixes:
 
       T, t    The size is expressed in tebibytes.
 
-      %       The size is expressed as a percentage of the free space on the
-              file system where it resides.
-
-The default size is to use 50% of the space available on the device.
+If 'path' points to an existing file and no size is specified, the
+size of the existing file will be used. If 'path' does not point to an
+existing file it is an error to not specify the size.
 
 If the backing file already exists, it will be truncated or expanded
 to the specified size.



From martin at varnish-software.com  Mon Jun  1 12:25:06 2015
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 01 Jun 2015 14:25:06 +0200
Subject: [4.0] 0efe5d9 Another mention of % in -sfile docs
Message-ID: <E1YzOmA-0006RR-2z@project.varnish-software.com>


commit 0efe5d94b1fa6f18ff23b8846882420dcb1f0dc9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 1 14:23:09 2015 +0200

    Another mention of % in -sfile docs

diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst
index 3539174..5d08d1e 100644
--- a/doc/sphinx/users-guide/storage-backends.rst
+++ b/doc/sphinx/users-guide/storage-backends.rst
@@ -86,7 +86,7 @@ the storage file using `dd(1)` will reduce fragmentation to a minimum.
 The 'granularity' parameter specifies the granularity of
 allocation. All allocations are rounded up to this size. The granularity is
 is assumed to be expressed in bytes, unless followed by one of the
-suffixes described for size except for %.
+suffixes described for size.
 
 The default granularity is the VM page size. The size should be reduced if you
 have many small objects.



From phk at FreeBSD.org  Mon Jun  1 19:36:09 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 01 Jun 2015 21:36:09 +0200
Subject: [master] 9b40808 Only bother/do g[un]zip if there actually is a
 beresp.body.
Message-ID: <E1YzVVJ-0000rV-8I@project.varnish-software.com>


commit 9b4080814677ae45cb858b97900046d96226c7d9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 1 19:35:03 2015 +0000

    Only bother/do g[un]zip if there actually is a beresp.body.
    
    Fixes:	#1746

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index b52354a..92a9c7f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -563,19 +563,24 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->htc->content_length == 0)
 		http_Unset(bo->beresp, H_Content_Encoding);
 
-	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
-
-	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
-
-	/* It can't be both */
-	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
+	if (bo->htc->body_status != BS_NONE) {
+		bo->is_gzip =
+		    http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
+		bo->is_gunzip =
+		    !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
+		assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
+	}
 
 	/* We won't gunzip unless it is non-empty and gzip'ed */
-	if (bo->htc->content_length == 0 || (bo->do_gunzip && !bo->is_gzip))
+	if (bo->htc->body_status == BS_NONE ||
+	    bo->htc->content_length == 0 ||
+	    (bo->do_gunzip && !bo->is_gzip))
 		bo->do_gunzip = 0;
 
 	/* We wont gzip unless it is non-empty and ungziped */
-	if (bo->htc->content_length == 0 || (bo->do_gzip && !bo->is_gunzip))
+	if (bo->htc->body_status == BS_NONE ||
+	    bo->htc->content_length == 0 ||
+	    (bo->do_gzip && !bo->is_gunzip))
 		bo->do_gzip = 0;
 
 	/* But we can't do both at the same time */
diff --git a/bin/varnishtest/tests/r01746.vtc b/bin/varnishtest/tests/r01746.vtc
new file mode 100644
index 0000000..0cc92f6
--- /dev/null
+++ b/bin/varnishtest/tests/r01746.vtc
@@ -0,0 +1,23 @@
+varnishtest "ESI include is 204"
+
+server s1 {
+	rxreq
+	txresp -gzipbody {<A><esi:include src="/b"/></A>}
+	rxreq
+	expect req.url == /b
+	txresp -status 204 -nolen \
+	    -hdr "Content-encoding: gzip" \
+	    -hdr "content-length: 10"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.body == "<A></A>"
+} -run



From phk at FreeBSD.org  Wed Jun  3 15:22:00 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 03 Jun 2015 17:22:00 +0200
Subject: [master] cbab1e1 Remove unused lock-families.
Message-ID: <E1Z0AUS-0006zB-GY@project.varnish-software.com>


commit cbab1e16bb0093ecb76c35e09c9207e2fe3b29af
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 3 15:02:42 2015 +0000

    Remove unused lock-families.

diff --git a/bin/varnishtest/tests/r01746.vtc b/bin/varnishtest/tests/r01746.vtc
index 0cc92f6..30c6137 100644
--- a/bin/varnishtest/tests/r01746.vtc
+++ b/bin/varnishtest/tests/r01746.vtc
@@ -21,3 +21,5 @@ client c1 {
 	rxresp
 	expect resp.body == "<A></A>"
 } -run
+
+varnish v1 -expect esi_errors == 0
diff --git a/include/tbl/locks.h b/include/tbl/locks.h
index 010de43..7331493 100644
--- a/include/tbl/locks.h
+++ b/include/tbl/locks.h
@@ -28,32 +28,27 @@
  */
 
 /*lint -save -e525 -e539 */
-LOCK(sms)
-LOCK(smp)
-LOCK(sma)
-LOCK(smf)
-LOCK(hsl)
+LOCK(backend)
+LOCK(ban)
+LOCK(busyobj)
+LOCK(cli)
+LOCK(exp)
 LOCK(hcb)
 LOCK(hcl)
-LOCK(vcl)
-LOCK(sessmem)
-LOCK(sess)
-LOCK(wstat)
-LOCK(herder)
-LOCK(wq)
-LOCK(objhdr)
-LOCK(exp)
+LOCK(hsl)
 LOCK(lru)
-LOCK(cli)
-LOCK(ban)
+LOCK(mempool)
+LOCK(misc)
+LOCK(objhdr)
+LOCK(pipestat)
+LOCK(sess)
+LOCK(sma)
+LOCK(smf)
+LOCK(smp)
 LOCK(vbe)
-LOCK(vbp)
-LOCK(backend)
 LOCK(vcapace)
-LOCK(nbusyobj)
-LOCK(busyobj)
-LOCK(mempool)
+LOCK(vcl)
 LOCK(vxid)
-LOCK(pipestat)
-LOCK(misc)
+LOCK(wq)
+LOCK(wstat)
 /*lint -restore */



From dridi.boukelmoune at gmail.com  Thu Jun  4 17:02:14 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 04 Jun 2015 19:02:14 +0200
Subject: [master] 190cbca sed s/Server/Client/ bin/varnishtest/vtc_client.c
Message-ID: <E1Z0YX0-0007u5-HU@project.varnish-software.com>


commit 190cbcada1c95f7eff6f6ebe530cce35e14c7b16
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu May 28 20:47:41 2015 +0200

    sed s/Server/Client/ bin/varnishtest/vtc_client.c

diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 9403af2..70e6cae 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -61,7 +61,7 @@ static VTAILQ_HEAD(, client)	clients =
     VTAILQ_HEAD_INITIALIZER(clients);
 
 /**********************************************************************
- * Server thread
+ * Client thread
  */
 
 static void *
@@ -196,7 +196,7 @@ client_run(struct client *c)
 
 
 /**********************************************************************
- * Server command dispatch
+ * Client command dispatch
  */
 
 void



From dridi.boukelmoune at gmail.com  Thu Jun  4 17:02:14 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 04 Jun 2015 19:02:14 +0200
Subject: [master] 17c2005 Add missing test category: prOxy protocol
Message-ID: <E1Z0YX0-0007uB-Ip@project.varnish-software.com>


commit 17c2005b655be82f7d9cb4a5b3502907ed8c44c4
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Sat May 30 11:43:35 2015 +0200

    Add missing test category: prOxy protocol

diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README
index 3d6fdca..456bf75 100644
--- a/bin/varnishtest/tests/README
+++ b/bin/varnishtest/tests/README
@@ -22,6 +22,7 @@ Naming scheme
 	id ~ [j] --> JAIL tests
 	id ~ [l] --> VSL tests
 	id ~ [m] --> VMOD tests excluding director
+	id ~ [o] --> prOxy protocol
 	id ~ [p] --> Persistent tests
 	id ~ [r] --> Regression tests, same number as ticket
 	id ~ [s] --> Slow tests, expiry, grace etc.



From dridi.boukelmoune at gmail.com  Thu Jun  4 17:02:14 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 04 Jun 2015 19:02:14 +0200
Subject: [master] 98d13b4 Remove duplicate -d varnishd option in varnishtest
Message-ID: <E1Z0YX0-0007uN-K7@project.varnish-software.com>


commit 98d13b40bab1a01107184f9581097ec4194f41da
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Sat May 30 16:34:38 2015 +0200

    Remove duplicate -d varnishd option in varnishtest

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index a50166e..a7b504f 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -392,7 +392,7 @@ varnish_launch(struct varnish *v)
 	vsb = VSB_new_auto();
 	AN(vsb);
 	VSB_printf(vsb, "cd ${pwd} &&");
-	VSB_printf(vsb, " exec ${varnishd} %s -d -d -n %s",
+	VSB_printf(vsb, " exec ${varnishd} %s -d -n %s",
 	    v->jail, v->workdir);
 	VSB_printf(vsb, " -l 2m,1m,-");
 	VSB_printf(vsb, " -p auto_restart=off");
@@ -763,7 +763,8 @@ do_stat_cb(void *priv, const struct VSC_point * const pt)
 }
 
 static void
-varnish_expect(const struct varnish *v, char * const *av) {
+varnish_expect(const struct varnish *v, char * const *av)
+{
 	uint64_t ref;
 	int good;
 	char *r;



From lkarsten at varnish-software.com  Mon Jun  8 12:34:54 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 08 Jun 2015 14:34:54 +0200
Subject: [4.0] 2252591 Add python as requirement for varnish-libs-devel
Message-ID: <E1Z1wGU-0007t1-4d@project.varnish-software.com>


commit 225259147e8f3cf46d7ba89145a69fe1f4e14178
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jun 8 14:30:22 2015 +0200

    Add python as requirement for varnish-libs-devel
    
    Follows from the discussion of 49712a5 on varnish-dev at .
    
    The development package contains vmodtool.py, which works better
    if the interpreter is available.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 0730497..4720c11 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -69,6 +69,7 @@ Summary: Development files for %{name}-libs
 Group: System Environment/Libraries
 BuildRequires: ncurses-devel
 Requires: varnish-libs = %{version}-%{release}
+Requires: python
 
 %description libs-devel
 Development files for %{name}-libs



From lkarsten at varnish-software.com  Mon Jun  8 12:34:59 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 08 Jun 2015 14:34:59 +0200
Subject: [master] 0fd62ec Add python as requirement for varnish-libs-devel
Message-ID: <E1Z1wGZ-0007wV-90@project.varnish-software.com>


commit 0fd62ece923b65765958e0eee4788e6c0a9dc4e9
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jun 8 14:30:22 2015 +0200

    Add python as requirement for varnish-libs-devel
    
    Follows from the discussion of 49712a5 on varnish-dev at .
    
    The development package contains vmodtool.py, which works better
    if the interpreter is available.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 2f5e5dd..9bce64d 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -75,6 +75,7 @@ Summary: Development files for %{name}-libs
 Group: System Environment/Libraries
 BuildRequires: ncurses-devel
 Requires: varnish-libs = %{version}-%{release}
+Requires: python
 
 %description libs-devel
 Development files for %{name}-libs



From lkarsten at varnish-software.com  Mon Jun  8 12:42:54 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 08 Jun 2015 14:42:54 +0200
Subject: [4.0] 7e3c3fe Prepare packages with new -libs-devel dependency.
Message-ID: <E1Z1wOE-0000Nx-E0@project.varnish-software.com>


commit 7e3c3fe94127dcfceb73401c0a3a3a18ad95289e
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jun 8 14:41:59 2015 +0200

    Prepare packages with new -libs-devel dependency.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 4720c11..fc9ac28 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -6,7 +6,7 @@ Summary: High-performance HTTP accelerator
 Name: varnish
 Version: 4.0.3
 #Release: 0.20140328%{?v_rc}%{?dist}
-Release: 1%{?v_rc}%{?dist}
+Release: 2%{?v_rc}%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: https://www.varnish-cache.org/



From lkarsten at varnish-software.com  Mon Jun  8 12:48:27 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 8 Jun 2015 14:48:27 +0200
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <CABoVN9B4+JznUDaKHpupVPWVw7fD3eN8JLEDg9eTZ32QwQ5E1g@mail.gmail.com>
References: <555E5690.8090706@schokola.de>
 <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>
 <555F2F8D.9030402@uplex.de>
 <CAJV_h0bvYpJ3DuQ+-z-jP_f+wkkOQyLY9pjdmP4bs+WyjSA9rw@mail.gmail.com>
 <5561A55B.4040608@uplex.de> <35990.1432465498@critter.freebsd.dk>
 <CA+nLBF0hNwLoTbEpQ=aP27KbiVbV6f-Uuc3Ffh-_e-2Q5_-1yg@mail.gmail.com>
 <CABoVN9AO-f6n0X=_TEaZQMZi61JPVhSDB47XiHthVpuMiavEmQ@mail.gmail.com>
 <CAJV_h0YL91CVT2pKkdkxDyMpn8xx6MAwMicR1+SkPnoPXYp2=Q@mail.gmail.com>
 <CABoVN9B4+JznUDaKHpupVPWVw7fD3eN8JLEDg9eTZ32QwQ5E1g@mail.gmail.com>
Message-ID: <20150608124826.GA4317@immer.varnish-software.com>

On Wed, May 27, 2015 at 12:56:28AM +0200, Dridi Boukelmoune wrote:
> On Tue, May 26, 2015 at 10:31 PM, Federico Schwindt <fgsch at lodoss.net> wrote:
> > I'd say that's a bug.
> Agreed, I have file an issue for Fedora:
> https://bugzilla.redhat.com/show_bug.cgi?id=1225243

Good find. I've fixed this in our EL packaging now.

Due to how the build pipeline for VC4 RPMs is set up, I won't be be
deploying updated RPMs right now. Fixed in git.

-- 
Lasse Karstensen
Varnish Software AS



From arianna.aondio at varnish-software.com  Mon Jun  8 13:40:03 2015
From: arianna.aondio at varnish-software.com (Arianna Aondio)
Date: Mon, 08 Jun 2015 15:40:03 +0200
Subject: [4.0] ea5fd91 Remove varnishreplay from the index.
Message-ID: <E1Z1xHX-00058z-B5@project.varnish-software.com>


commit ea5fd91d7b91e4c71215075861820628a7345919
Author: Arianna Aondio <arianna.aondio at varnish-software.com>
Date:   Mon Jun 8 15:39:55 2015 +0200

    Remove varnishreplay from the index.

diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index b59beb5..014289c 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -14,7 +14,6 @@ The Varnish Reference Manual
 	varnishhist.rst
 	varnishlog.rst
 	varnishncsa.rst
-	varnishreplay.rst
 	varnishstat.rst
 	varnishtest.rst
 	varnishtop.rst



From arianna.aondio at varnish-software.com  Mon Jun  8 13:49:05 2015
From: arianna.aondio at varnish-software.com (Arianna Aondio)
Date: Mon, 08 Jun 2015 15:49:05 +0200
Subject: [4.0] bfbaae3 Remove varnishreplay.rst, it is no more needed.
Message-ID: <E1Z1xQH-00085b-Uo@project.varnish-software.com>


commit bfbaae3bd4e56386144d223827f7bdc8a2dba43f
Author: Arianna Aondio <arianna.aondio at varnish-software.com>
Date:   Mon Jun 8 15:47:40 2015 +0200

    Remove varnishreplay.rst, it is no more needed.

diff --git a/doc/sphinx/reference/varnishreplay.rst b/doc/sphinx/reference/varnishreplay.rst
deleted file mode 100644
index 884e7b7..0000000
--- a/doc/sphinx/reference/varnishreplay.rst
+++ /dev/null
@@ -1,50 +0,0 @@
-=============
-varnishreplay
-=============
-
-------------------------
-HTTP traffic replay tool
-------------------------
-
-SYNOPSIS
-========
-varnishreplay [-D] -a address:port -r file
-
-DESCRIPTION
-===========
-
-The varnishreplay utility parses Varnish logs and attempts to
-reproduce the traffic. It is typically used to *warm* up caches or
-various forms of testing.
-
-The following options are available:
-
--a backend           Send the traffic over tcp to this server, specified by an
-   		     address and a port.  This option is
-   		     mandatory. Only IPV4 is supported at this time.
-
--D                   Turn on debugging mode.
-
--r file              Parse logs from this file.  The input file has to be from
-		     a varnishlog of the same version as the varnishreplay
-		     binary.  This option is mandatory.
-
-SEE ALSO
-========
-
-* varnishd(1)
-* varnishlog(1)
-
-HISTORY
-=======
-
-The varnishreplay utility and this manual page were written by Cecilie
-Fritzvold and later updated by Per Buer.
-
-COPYRIGHT
-=========
-
-This document is licensed under the same licence as Varnish
-itself. See LICENCE for details.
-
-* Copyright (c) 2007-2014 Varnish Software AS



From dridi.boukelmoune at gmail.com  Tue Jun  9 09:01:11 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 09 Jun 2015 11:01:11 +0200
Subject: [master] acb98c8 Fix the -t option formatting for Varnish tools
Message-ID: <E1Z2FPD-0000D6-Rf@project.varnish-software.com>


commit acb98c8f779ec433ffa00d41cf1503934726d0a8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jun 9 10:51:52 2015 +0200

    Fix the -t option formatting for Varnish tools
    
    It didn't look like the other options in both man pages and sphinx docs.
    However, options like "-t <...>" are still somewhat broken in the sphinx
    documentation: the option and description are on different rows.

diff --git a/include/vut_options.h b/include/vut_options.h
index 6004690..c1304d6 100644
--- a/include/vut_options.h
+++ b/include/vut_options.h
@@ -85,7 +85,7 @@
 	)
 
 #define VUT_OPT_t							\
-	VOPT("t:", "[-t seconds|<off>]", "VSM connection timeout",	\
+	VOPT("t:", "[-t <seconds|off>]", "VSM connection timeout",	\
 	    "Timeout before returning error on initial VSM connection."	\
 	    " If set the VSM connection is retried every 0.5 seconds"	\
 	    " for this many seconds. If zero the connection is"		\



From dridi.boukelmoune at gmail.com  Tue Jun  9 15:24:35 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 09 Jun 2015 17:24:35 +0200
Subject: [4.0] 299bc3d Document varnishlog -w/-r with more details
Message-ID: <E1Z2LOF-0003hw-5l@project.varnish-software.com>


commit 299bc3d6d0f12994a5722ee5b40279a0bba94c76
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jun 9 17:20:41 2015 +0200

    Document varnishlog -w/-r with more details
    
    Fixes #1742

diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h
index 55b30c7..f97642a 100644
--- a/bin/varnishlog/varnishlog_options.h
+++ b/bin/varnishlog/varnishlog_options.h
@@ -38,7 +38,8 @@
 
 #define LOG_OPT_B							\
 	VOPT("B", "[-B]", "Binary output",				\
-	    "Output binary data suitable for reading with -r."		\
+	    "Output binary data suitable for reading with -r. The -w"	\
+	    " option specifies where the binary output is written."	\
 	)
 
 #define LOG_OPT_w							\
diff --git a/include/vut_options.h b/include/vut_options.h
index 9f01181..0a69254 100644
--- a/include/vut_options.h
+++ b/include/vut_options.h
@@ -81,7 +81,8 @@
 
 #define VUT_OPT_r							\
 	VOPT("r:", "[-r filename]", "Binary file input",		\
-	    "Read log in binary file format from this file."		\
+	    "Read log in binary file format from this file. The file"   \
+	    " can be created with ``varnishlog -B -w filename``."	\
 	)
 
 #define VUT_OPT_V							\



From phk at FreeBSD.org  Tue Jun  9 20:31:25 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 09 Jun 2015 22:31:25 +0200
Subject: [master] d5db14b Retire POOL_NO_QUEUE, we don't use it.
Message-ID: <E1Z2QBB-0004im-Aq@project.varnish-software.com>


commit d5db14b2cf75c65ec5b298d1ddf8d6c4410c346d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 19:44:54 2015 +0000

    Retire POOL_NO_QUEUE, we don't use it.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 3c38a04..72a61f9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -314,7 +314,6 @@ struct pool_task {
 };
 
 enum pool_how {
-	POOL_NO_QUEUE,
 	POOL_QUEUE_FRONT,
 	POOL_QUEUE_BACK
 };
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index ee8bd5e..dd370e0 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -245,9 +245,6 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	}
 
 	switch (how) {
-	case POOL_NO_QUEUE:
-		retval = -1;
-		break;
 	case POOL_QUEUE_FRONT:
 		/* If we have too much in the queue already, refuse. */
 		if (pp->lqueue > cache_param->wthread_queue_limit) {



From phk at FreeBSD.org  Tue Jun  9 20:31:25 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 09 Jun 2015 22:31:25 +0200
Subject: [master] a84f28e Rename to more appropriate names: It's about the
 tasks not the pools that service them.
Message-ID: <E1Z2QBB-0004it-CN@project.varnish-software.com>


commit a84f28ee669ed7fb983de6a5a7d6d1b1bb2972e0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 19:48:05 2015 +0000

    Rename to more appropriate names:  It's about the tasks not the
    pools that service them.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 72a61f9..890ebe2 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -313,9 +313,9 @@ struct pool_task {
 	void				*priv;
 };
 
-enum pool_how {
-	POOL_QUEUE_FRONT,
-	POOL_QUEUE_BACK
+enum task_how {
+	TASK_QUEUE_FRONT,
+	TASK_QUEUE_BACK
 };
 
 /*--------------------------------------------------------------------*/
@@ -946,13 +946,13 @@ const char *sess_close_2str(enum sess_close sc, int want_desc);
 
 /* cache_pool.c */
 void Pool_Init(void);
-int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how);
+int Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how);
 int Pool_Task_Arg(struct worker *, task_func_t *,
     const void *arg, size_t arg_len);
 void Pool_Sumstat(struct worker *w);
 int Pool_TrySumstat(struct worker *wrk);
 void Pool_PurgeStat(unsigned nobj);
-int Pool_Task_Any(struct pool_task *task, enum pool_how how);
+int Pool_Task_Any(struct pool_task *task, enum task_how how);
 
 #define V1L_IsReleased(w)	((w)->v1l == NULL)
 void V1L_Chunked(const struct worker *w);
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 45a1828..479754e 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -437,7 +437,7 @@ vca_accept_task(struct worker *wrk, void *arg)
 			 * must reschedule the listening task so it will be
 			 * taken up by another thread again.
 			 */
-			AZ(Pool_Task(wrk->pool, &ps->task, POOL_QUEUE_BACK));
+			AZ(Pool_Task(wrk->pool, &ps->task, TASK_QUEUE_BACK));
 			return;
 		}
 
@@ -468,7 +468,7 @@ VCA_NewPool(struct pool *pp)
 		ps->task.func = vca_accept_task;
 		ps->task.priv = ps;
 		ps->pool = pp;
-		AZ(Pool_Task(pp, &ps->task, POOL_QUEUE_BACK));
+		AZ(Pool_Task(pp, &ps->task, TASK_QUEUE_BACK));
 	}
 }
 
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index 641ed4e..0d06693 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -363,7 +363,7 @@ vbp_thread(struct worker *wrk, void *priv)
 		vt->task.func = vbp_task;
 		vt->task.priv = vt;
 
-		if (Pool_Task_Any(&vt->task, POOL_QUEUE_FRONT)) {
+		if (Pool_Task_Any(&vt->task, TASK_QUEUE_FRONT)) {
 			Lck_Lock(&vbp_mtx);
 			vt->running = 0;
 			Lck_Unlock(&vbp_mtx);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 92a9c7f..73ad0a4 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -1003,7 +1003,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
 	bo->fetch_task.priv = bo_fetch;
 	bo->fetch_task.func = vbf_fetch_thread;
 
-	if (Pool_Task(wrk->pool, &bo->fetch_task, POOL_QUEUE_FRONT)) {
+	if (Pool_Task(wrk->pool, &bo->fetch_task, TASK_QUEUE_FRONT)) {
 		wrk->stats->fetch_no_thread++;
 		(void)vbf_stp_fail(req->wrk, bo);
 		if (bo->stale_oc != NULL)
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 56f11bf..f02e594 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -91,7 +91,7 @@ Pool_TrySumstat(struct worker *wrk)
  */
 
 int
-Pool_Task_Any(struct pool_task *task, enum pool_how how)
+Pool_Task_Any(struct pool_task *task, enum task_how how)
 {
 	struct pool *pp;
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 9820758..30f4b08 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -415,7 +415,7 @@ SES_Reschedule_Req(struct req *req)
 	req->task.func = SES_Proto_Req;
 	req->task.priv = req;
 
-	return (Pool_Task(pp, &req->task, POOL_QUEUE_FRONT));
+	return (Pool_Task(pp, &req->task, TASK_QUEUE_FRONT));
 }
 
 /*--------------------------------------------------------------------
@@ -451,7 +451,7 @@ ses_handle(struct waited *wp, enum wait_event ev, double now)
 		tp = (void*)sp->ws->f;
 		tp->func = SES_Proto_Sess;
 		tp->priv = sp;
-		if (Pool_Task(pp, tp, POOL_QUEUE_FRONT))
+		if (Pool_Task(pp, tp, TASK_QUEUE_FRONT))
 			SES_Delete(sp, SC_OVERLOAD, now);
 		break;
 	case WAITER_CLOSE:
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index dd370e0..466b7b9 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -218,7 +218,7 @@ Pool_Task_Arg(struct worker *wrk, task_func_t *func,
  */
 
 int
-Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
+Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how)
 {
 	struct worker *wrk;
 	int retval = 0;
@@ -245,7 +245,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	}
 
 	switch (how) {
-	case POOL_QUEUE_FRONT:
+	case TASK_QUEUE_FRONT:
 		/* If we have too much in the queue already, refuse. */
 		if (pp->lqueue > cache_param->wthread_queue_limit) {
 			pp->ndropped++;
@@ -256,11 +256,11 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 			pp->lqueue++;
 		}
 		break;
-	case POOL_QUEUE_BACK:
+	case TASK_QUEUE_BACK:
 		VTAILQ_INSERT_TAIL(&pp->back_queue, task, list);
 		break;
 	default:
-		WRONG("Unknown enum pool_how");
+		WRONG("Unknown enum task_how");
 	}
 	Lck_Unlock(&pp->mtx);
 	return (retval);



From phk at FreeBSD.org  Tue Jun  9 20:31:25 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 09 Jun 2015 22:31:25 +0200
Subject: [master] 067662b Prepare for having more than two task queues,
 by turning them into an array of queues.
Message-ID: <E1Z2QBB-0004j2-DR@project.varnish-software.com>


commit 067662b568655de3a8e4f940322aee54d678a2fe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 19:59:51 2015 +0000

    Prepare for having more than two task queues, by turning them into
    an array of queues.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 890ebe2..e36d9b7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -315,7 +315,8 @@ struct pool_task {
 
 enum task_how {
 	TASK_QUEUE_FRONT,
-	TASK_QUEUE_BACK
+	TASK_QUEUE_BACK,
+	TASK_QUEUE_END
 };
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index f02e594..d3deed2 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -149,6 +149,7 @@ static struct pool *
 pool_mkpool(unsigned pool_no)
 {
 	struct pool *pp;
+	int i;
 
 	ALLOC_OBJ(pp, POOL_MAGIC);
 	if (pp == NULL)
@@ -160,8 +161,8 @@ pool_mkpool(unsigned pool_no)
 	Lck_New(&pp->mtx, lck_wq);
 
 	VTAILQ_INIT(&pp->idle_queue);
-	VTAILQ_INIT(&pp->front_queue);
-	VTAILQ_INIT(&pp->back_queue);
+	for (i = 0; i < TASK_QUEUE_END; i++)
+		VTAILQ_INIT(&pp->queues[i]);
 	AZ(pthread_cond_init(&pp->herder_cond, NULL));
 	AZ(pthread_create(&pp->herder_thr, NULL, pool_herder, pp));
 
diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
index ea343e6..d3b9e70 100644
--- a/bin/varnishd/cache/cache_pool.h
+++ b/bin/varnishd/cache/cache_pool.h
@@ -43,8 +43,7 @@ struct pool {
 
 	struct lock			mtx;
 	struct taskhead			idle_queue;
-	struct taskhead			front_queue;
-	struct taskhead			back_queue;
+	struct taskhead			queues[TASK_QUEUE_END];
 	unsigned			nthr;
 	unsigned			dry;
 	unsigned			lqueue;
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 466b7b9..a76f02b 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -251,13 +251,14 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how)
 			pp->ndropped++;
 			retval = -1;
 		} else {
-			VTAILQ_INSERT_TAIL(&pp->front_queue, task, list);
+			VTAILQ_INSERT_TAIL(&pp->queues[how], task, list);
 			pp->nqueued++;
 			pp->lqueue++;
 		}
 		break;
 	case TASK_QUEUE_BACK:
-		VTAILQ_INSERT_TAIL(&pp->back_queue, task, list);
+		VTAILQ_INSERT_TAIL(&pp->queues[how], task, list);
+		pp->lqueue++;
 		break;
 	default:
 		WRONG("Unknown enum task_how");
@@ -299,14 +300,13 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 		WS_Reset(wrk->aws, NULL);
 		AZ(wrk->vsl);
 
-		tp = VTAILQ_FIRST(&pp->front_queue);
-		if (tp != NULL) {
-			pp->lqueue--;
-			VTAILQ_REMOVE(&pp->front_queue, tp, list);
-		} else {
-			tp = VTAILQ_FIRST(&pp->back_queue);
-			if (tp != NULL)
-				VTAILQ_REMOVE(&pp->back_queue, tp, list);
+		for (i = 0; i < TASK_QUEUE_END; i++) {
+			tp = VTAILQ_FIRST(&pp->queues[i]);
+			if (tp != NULL) {
+				pp->lqueue--;
+				VTAILQ_REMOVE(&pp->queues[i], tp, list);
+				break;
+			}
 		}
 
 		if ((tp == NULL && wrk->stats->summs > 0) ||



From phk at FreeBSD.org  Tue Jun  9 20:31:25 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 09 Jun 2015 22:31:25 +0200
Subject: [master] 42417e3 Generalize the task queuing code and specifically
 exempt acceptor tasks from the queue limits.
Message-ID: <E1Z2QBB-0004jB-Es@project.varnish-software.com>


commit 42417e3631559b01f827e1c455c898d7a8a019d6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 20:17:42 2015 +0000

    Generalize the task queuing code and specifically exempt acceptor
    tasks from the queue limits.
    
    Slightly change the queuelimits to be relative to the full complement
    of worker threads in the pool, rather than the count of currently created
    threads, this should drop less traffic during ramp-up, but retain the
    steady-state behaviour.

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index a76f02b..2337ab9 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -226,12 +226,11 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how)
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	AN(task);
 	AN(task->func);
+	assert(how >= TASK_QUEUE_FRONT && how <= TASK_QUEUE_BACK);
 
 	Lck_Lock(&pp->mtx);
 
-	/*
-	 * The common case first:  Take an idle thread, do it.
-	 */
+	/* The common case first:  Take an idle thread, do it. */
 
 	wrk = pool_getidleworker(pp);
 	if (wrk != NULL) {
@@ -244,24 +243,16 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how)
 		return (0);
 	}
 
-	switch (how) {
-	case TASK_QUEUE_FRONT:
-		/* If we have too much in the queue already, refuse. */
-		if (pp->lqueue > cache_param->wthread_queue_limit) {
-			pp->ndropped++;
-			retval = -1;
-		} else {
-			VTAILQ_INSERT_TAIL(&pp->queues[how], task, list);
-			pp->nqueued++;
-			pp->lqueue++;
-		}
-		break;
-	case TASK_QUEUE_BACK:
-		VTAILQ_INSERT_TAIL(&pp->queues[how], task, list);
+	/* Acceptors are not subject to queue limits */
+	if (how == TASK_QUEUE_BACK ||
+	    pp->lqueue < cache_param->wthread_max +
+	    cache_param->wthread_queue_limit + pp->nthr) {
+		pp->nqueued++;
 		pp->lqueue++;
-		break;
-	default:
-		WRONG("Unknown enum task_how");
+		VTAILQ_INSERT_TAIL(&pp->queues[how], task, list);
+	} else {
+		pp->ndropped++;
+		retval = -1;
 	}
 	Lck_Unlock(&pp->mtx);
 	return (retval);



From phk at FreeBSD.org  Tue Jun  9 20:31:25 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 09 Jun 2015 22:31:25 +0200
Subject: [master] 00871ce Give TASK_QUEUE_ enum names decriptive of what is
 being queued (BO, REQ & VCA)
Message-ID: <E1Z2QBB-0004jK-G3@project.varnish-software.com>


commit 00871cef5322f39807c819e52a00013703e2bdb4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 20:28:05 2015 +0000

    Give TASK_QUEUE_ enum names decriptive of what is being queued
    (BO, REQ & VCA)
    
    Add new a new TASK_QUEUE level for backend requests and give it
    higher priority than client requests to avoid a priority inversion
    when (almost) all threads are stuck waiting for backend fetches
    which doesn't happen because there are no worker threads.
    
    Backend polls are left at TASK_QUEUE_REQ priority for now, pending
    reasons not to.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index e36d9b7..a29d998 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -314,8 +314,9 @@ struct pool_task {
 };
 
 enum task_how {
-	TASK_QUEUE_FRONT,
-	TASK_QUEUE_BACK,
+	TASK_QUEUE_BO,
+	TASK_QUEUE_REQ,
+	TASK_QUEUE_VCA,
 	TASK_QUEUE_END
 };
 
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 479754e..90965c3 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -437,7 +437,7 @@ vca_accept_task(struct worker *wrk, void *arg)
 			 * must reschedule the listening task so it will be
 			 * taken up by another thread again.
 			 */
-			AZ(Pool_Task(wrk->pool, &ps->task, TASK_QUEUE_BACK));
+			AZ(Pool_Task(wrk->pool, &ps->task, TASK_QUEUE_VCA));
 			return;
 		}
 
@@ -468,7 +468,7 @@ VCA_NewPool(struct pool *pp)
 		ps->task.func = vca_accept_task;
 		ps->task.priv = ps;
 		ps->pool = pp;
-		AZ(Pool_Task(pp, &ps->task, TASK_QUEUE_BACK));
+		AZ(Pool_Task(pp, &ps->task, TASK_QUEUE_VCA));
 	}
 }
 
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index 0d06693..c658417 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -363,7 +363,7 @@ vbp_thread(struct worker *wrk, void *priv)
 		vt->task.func = vbp_task;
 		vt->task.priv = vt;
 
-		if (Pool_Task_Any(&vt->task, TASK_QUEUE_FRONT)) {
+		if (Pool_Task_Any(&vt->task, TASK_QUEUE_REQ)) {
 			Lck_Lock(&vbp_mtx);
 			vt->running = 0;
 			Lck_Unlock(&vbp_mtx);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 73ad0a4..3e13dd2 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -1003,7 +1003,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
 	bo->fetch_task.priv = bo_fetch;
 	bo->fetch_task.func = vbf_fetch_thread;
 
-	if (Pool_Task(wrk->pool, &bo->fetch_task, TASK_QUEUE_FRONT)) {
+	if (Pool_Task(wrk->pool, &bo->fetch_task, TASK_QUEUE_BO)) {
 		wrk->stats->fetch_no_thread++;
 		(void)vbf_stp_fail(req->wrk, bo);
 		if (bo->stale_oc != NULL)
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 30f4b08..e19c0b9 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -415,7 +415,7 @@ SES_Reschedule_Req(struct req *req)
 	req->task.func = SES_Proto_Req;
 	req->task.priv = req;
 
-	return (Pool_Task(pp, &req->task, TASK_QUEUE_FRONT));
+	return (Pool_Task(pp, &req->task, TASK_QUEUE_REQ));
 }
 
 /*--------------------------------------------------------------------
@@ -451,7 +451,7 @@ ses_handle(struct waited *wp, enum wait_event ev, double now)
 		tp = (void*)sp->ws->f;
 		tp->func = SES_Proto_Sess;
 		tp->priv = sp;
-		if (Pool_Task(pp, tp, TASK_QUEUE_FRONT))
+		if (Pool_Task(pp, tp, TASK_QUEUE_REQ))
 			SES_Delete(sp, SC_OVERLOAD, now);
 		break;
 	case WAITER_CLOSE:
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 2337ab9..7167303 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -226,7 +226,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how)
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	AN(task);
 	AN(task->func);
-	assert(how >= TASK_QUEUE_FRONT && how <= TASK_QUEUE_BACK);
+	assert(how >= TASK_QUEUE_BO && how <= TASK_QUEUE_VCA);
 
 	Lck_Lock(&pp->mtx);
 
@@ -244,7 +244,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how)
 	}
 
 	/* Acceptors are not subject to queue limits */
-	if (how == TASK_QUEUE_BACK ||
+	if (how == TASK_QUEUE_VCA ||
 	    pp->lqueue < cache_param->wthread_max +
 	    cache_param->wthread_queue_limit + pp->nthr) {
 		pp->nqueued++;



From phk at FreeBSD.org  Tue Jun  9 21:42:48 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 09 Jun 2015 23:42:48 +0200
Subject: [master] 454107b GCC doesn't let us assert the lower bound.
Message-ID: <E1Z2RIG-0002Ud-Pz@project.varnish-software.com>


commit 454107b77ea2509c9035d559a37c5fe079a080be
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 21:42:29 2015 +0000

    GCC doesn't let us assert the lower bound.

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 7167303..d9a34a7 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -226,7 +226,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how)
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	AN(task);
 	AN(task->func);
-	assert(how >= TASK_QUEUE_BO && how <= TASK_QUEUE_VCA);
+	assert(how <= TASK_QUEUE_VCA);
 
 	Lck_Lock(&pp->mtx);
 



From dridi.boukelmoune at gmail.com  Wed Jun 10 09:07:38 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 10 Jun 2015 11:07:38 +0200
Subject: [master] 72509ff Bound check on TASK_QUEUE_END instead
Message-ID: <E1Z2bz0-0003qE-Ob@project.varnish-software.com>


commit 72509ff650d5c203c07d8ad5f6fbe607cbb139a8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jun 10 11:05:38 2015 +0200

    Bound check on TASK_QUEUE_END instead

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index d9a34a7..86e0b92 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -226,7 +226,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_how how)
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	AN(task);
 	AN(task->func);
-	assert(how <= TASK_QUEUE_VCA);
+	assert(how < TASK_QUEUE_END);
 
 	Lck_Lock(&pp->mtx);
 



From dridi.boukelmoune at gmail.com  Wed Jun 10 09:49:30 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 10 Jun 2015 11:49:30 +0200
Subject: [master] 166ce40 Run background processes in varnishtest
Message-ID: <E1Z2cdW-00069m-VL@project.varnish-software.com>


commit 166ce403aaf7584041d195e885780d3f25f8f347
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Sat May 30 11:38:58 2015 +0200

    Run background processes in varnishtest
    
    Available operations on processes are:
    - start
    - wait
    - kill (send a signal)
    - stop (kill with TERM)
    - write (to its stdin)
    - close (its stdin)
    
    A process name starts with a 'p' and accepts a shell pipeline as the
    command line. Environment variables can be set directly in the command
    line.
    
    A process will create a macro with its PID, and its stdout and stderr
    are redirected to files in a dedicated directory.
    
    Also add a varnish "name" macro in varnishtest.

diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
index fe599d4..2506a8a 100644
--- a/bin/varnishtest/Makefile.am
+++ b/bin/varnishtest/Makefile.am
@@ -34,7 +34,8 @@ varnishtest_SOURCES = \
 		vtc_sema.c \
 		vtc_server.c \
 		vtc_varnish.c \
-		vtc_logexp.c
+		vtc_logexp.c \
+		vtc_process.c
 
 varnishtest_LDADD = \
 		$(top_builddir)/lib/libvarnish/libvarnish.la \
diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README
index 456bf75..52e5061 100644
--- a/bin/varnishtest/tests/README
+++ b/bin/varnishtest/tests/README
@@ -27,4 +27,5 @@ Naming scheme
 	id ~ [r] --> Regression tests, same number as ticket
 	id ~ [s] --> Slow tests, expiry, grace etc.
 	id ~ [t] --> sTreaming tests
+	id ~ [u] --> Unusual background processes
 	id ~ [v] --> VCL tests: execute VRT functions
diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
new file mode 100644
index 0000000..abe5f0e
--- /dev/null
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -0,0 +1,34 @@
+varnishtest "Simple process tests"
+
+# new & start
+process p1 "cat" -start
+process p2 "cat" -start
+process p3 "cat" -start
+
+# write
+process p1 -writeln "foo"
+process p2 -writeln "bar"
+process p3 -writeln "baz"
+
+# give enough time for the writes
+delay 0.5
+
+# stop
+process p1 -stop
+process p2 -close
+process p3 -kill "HUP"
+
+# wait
+process p1 -wait
+process p2 -wait
+process p3 -wait
+
+# check stdout
+shell "grep foo ${tmpdir}/p1/stdout >/dev/null 2>&1"
+shell "grep bar ${tmpdir}/p2/stdout >/dev/null 2>&1"
+shell "grep baz ${tmpdir}/p3/stdout >/dev/null 2>&1"
+
+# check stderr
+shell "test -f ${tmpdir}/p1/stderr -a ! -s ${tmpdir}/p1/stderr"
+shell "test -f ${tmpdir}/p2/stderr -a ! -s ${tmpdir}/p2/stderr"
+shell "test -f ${tmpdir}/p3/stderr -a ! -s ${tmpdir}/p3/stderr"
diff --git a/bin/varnishtest/tests/u00001.vtc b/bin/varnishtest/tests/u00001.vtc
new file mode 100644
index 0000000..7cf18c2
--- /dev/null
+++ b/bin/varnishtest/tests/u00001.vtc
@@ -0,0 +1,43 @@
+varnishtest "varnishncsa log file rotation"
+
+server s1 {
+	rxreq
+	expect req.url == "/foo"
+	txresp -status 200
+
+	rxreq
+	expect req.url == "/bar"
+	txresp -status 404
+} -start
+
+varnish v1 -vcl+backend "" -start
+
+process p1 "${varnishncsa} -n ${v1_name} -w ${tmpdir}/ncsa.log -F %s" -start
+
+# give varnishncsa enough time to open the VSM
+delay 1
+
+client c1 {
+	txreq -url "/foo"
+	rxresp
+} -run
+
+# give varnishncsa enough time to write
+delay 1
+
+# rotate logs
+shell "mv ${tmpdir}/ncsa.log ${tmpdir}/ncsa.old.log >/dev/null 2>&1"
+process p1 -kill "HUP"
+
+client c1 {
+	txreq -url "/bar"
+	rxresp
+} -run
+
+# give varnishncsa enough time to write
+delay 1
+
+process p1 -stop
+
+shell "grep 200 ${tmpdir}/ncsa.old.log >/dev/null"
+shell "grep 404 ${tmpdir}/ncsa.log >/dev/null"
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index cbda7bf..ae38f68 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -620,6 +620,7 @@ static const struct cmds cmds[] = {
 	{ "random",	cmd_random },
 	{ "feature",	cmd_feature },
 	{ "logexpect",	cmd_logexp },
+	{ "process",	cmd_process },
 	{ NULL,		NULL }
 };
 
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index e27c056..17b5790 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -63,6 +63,7 @@ cmd_f cmd_client;
 cmd_f cmd_varnish;
 cmd_f cmd_sema;
 cmd_f cmd_logexp;
+cmd_f cmd_process;
 
 extern volatile sig_atomic_t vtc_error; /* Error, bail out */
 extern int vtc_stop;		/* Abandon current test, no error */
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
new file mode 100644
index 0000000..157bfa9
--- /dev/null
+++ b/bin/varnishtest/vtc_process.c
@@ -0,0 +1,389 @@
+/*-
+ * Copyright (c) 2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dridi Boukelmoune <dridi at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <sys/resource.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "vtc.h"
+
+#include "vss.h"
+
+struct process {
+	unsigned		magic;
+#define PROCESS_MAGIC		0x1617b43e
+	char			*name;
+	struct vtclog		*vl;
+	VTAILQ_ENTRY(process)	list;
+
+	char			*spec;
+	char			*workdir;
+	char			*outdir;
+	char			*out;
+	char			*err;
+	int			fds[2];
+	pid_t			pid;
+
+	pthread_t		tp;
+	unsigned		running;
+	int			status;
+};
+
+static VTAILQ_HEAD(, process)	processes =
+    VTAILQ_HEAD_INITIALIZER(processes);
+
+/**********************************************************************
+ * Allocate and initialize a process
+ */
+
+#define PROCESS_EXPAND(field, format, ...)		\
+	do {						\
+		bprintf(buf, format, __VA_ARGS__);	\
+		vsb = macro_expand(p->vl, buf);		\
+		AN(vsb);				\
+		p->field = strdup(VSB_data(vsb));	\
+		AN(p->field);				\
+		VSB_delete(vsb);			\
+	} while (0)
+
+static struct process *
+process_new(const char *name)
+{
+	struct process *p;
+	struct vsb *vsb;
+	char buf[1024];
+
+	AN(name);
+	ALLOC_OBJ(p, PROCESS_MAGIC);
+	AN(p);
+	REPLACE(p->name, name);
+
+	p->vl = vtc_logopen(name);
+	AN(p->vl);
+
+	PROCESS_EXPAND(workdir, "%s", "${pwd}");
+	PROCESS_EXPAND(outdir, "${tmpdir}/%s", name);
+	PROCESS_EXPAND(out, "${tmpdir}/%s/stdout", name);
+	PROCESS_EXPAND(err, "${tmpdir}/%s/stderr", name);
+
+	bprintf(buf, "rm -rf %s ; mkdir -p %s ; touch %s %s",
+	    p->outdir, p->outdir, p->out, p->err);
+	AZ(system(buf));
+
+	p->fds[0] = -1;
+	p->fds[1] = -1;
+
+	if (*p->name != 'p')
+		vtc_log(p->vl, 0, "Process name must start with 'p'");
+
+	VTAILQ_INSERT_TAIL(&processes, p, list);
+	return (p);
+}
+
+#undef PROCESS_EXPAND
+
+/**********************************************************************
+ * Clean up process
+ */
+
+static void
+process_delete(struct process *p)
+{
+
+	CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC);
+	vtc_logclose(p->vl);
+	free(p->name);
+	free(p->workdir);
+	free(p->outdir);
+	free(p->out);
+	free(p->err);
+
+	/*
+	 * We do not delete the outdir, it may contain useful stdout
+	 * and stderr files.
+	 */
+
+	/* XXX: MEMLEAK (?) */
+	FREE_OBJ(p);
+}
+
+/**********************************************************************
+ * Start the process thread
+ */
+
+static void *
+process_thread(void *priv)
+{
+	struct process *p;
+	struct rusage ru;
+	int r;
+
+	CAST_OBJ_NOTNULL(p, priv, PROCESS_MAGIC);
+	r = wait4(p->pid, &p->status, 0, &ru);
+	macro_undef(p->vl, p->name, "pid");
+	p->pid = 0;
+	p->running = 0;
+	vtc_log(p->vl, 2, "R %d Status: %04x (u %.6f s %.6f)", r, p->status,
+	    ru.ru_utime.tv_sec + 1e-6 * ru.ru_utime.tv_usec,
+	    ru.ru_stime.tv_sec + 1e-6 * ru.ru_stime.tv_usec
+	);
+
+	if (WIFEXITED(p->status) && WEXITSTATUS(p->status) == 0)
+		return (NULL);
+#ifdef WCOREDUMP
+	vtc_log(p->vl, 2, "Bad exit code: %04x sig %x exit %x core %x",
+	    p->status, WTERMSIG(p->status), WEXITSTATUS(p->status),
+	    WCOREDUMP(p->status));
+#else
+	vtc_log(p->vl, 2, "Bad exit code: %04x sig %x exit %x",
+	    p->status, WTERMSIG(p->status), WEXITSTATUS(p->status));
+#endif
+
+	(void)close(p->fds[1]);
+	p->fds[1] = -1;
+
+	return (NULL);
+}
+
+static void
+process_start(struct process *p)
+{
+	struct vsb *cl;
+	int i, out_fd, err_fd;
+
+	CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC);
+
+	vtc_log(p->vl, 4, "CMD: %s", p->spec);
+
+	cl = macro_expand(p->vl, p->spec);
+	AN(cl);
+	AZ(pipe(p->fds));
+	out_fd = open(p->out, O_WRONLY|O_APPEND);
+	assert(out_fd >= 0);
+	err_fd = open(p->err, O_WRONLY|O_APPEND);
+	assert(err_fd >= 0);
+	p->pid = fork();
+	assert(p->pid >= 0);
+	p->running = 1;
+	if (p->pid == 0) {
+		assert(dup2(p->fds[0], 0) == 0);
+		assert(dup2(out_fd, 1) == 1);
+		assert(dup2(out_fd, 2) == 2);
+		for (i = 3; i <getdtablesize(); i++)
+			(void)close(i);
+		AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(cl), (char*)0));
+		exit(1);
+	}
+	vtc_log(p->vl, 3, "PID: %ld", (long)p->pid);
+	macro_def(p->vl, p->name, "pid", "%ld", (long)p->pid);
+	AZ(close(p->fds[0]));
+	AZ(close(out_fd));
+	AZ(close(err_fd));
+	p->fds[0] = -1;
+	VSB_delete(cl);
+	AZ(pthread_create(&p->tp, NULL, process_thread, p));
+}
+
+/**********************************************************************
+ * Wait for process thread to stop
+ */
+
+static void
+process_wait(struct process *p)
+{
+	void *v;
+
+	if (p->running && p->pid)
+		AZ(pthread_join(p->tp, &v));
+}
+
+/**********************************************************************
+ * Send a signal to a process
+ */
+
+static void
+process_kill(struct process *p, const char *sig)
+{
+	int s, l;
+	char buf[64];
+
+	CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC);
+	AN(sig);
+
+	if (!p->running || !p->pid) {
+		vtc_log(p->vl, 0, "Cannot signal a non-running process");
+		return;
+	}
+
+	vtc_log(p->vl, 4, "CMD: kill -%s %d", sig, p->pid);
+
+	l = snprintf(buf, sizeof buf, "kill -%s %d", sig, p->pid);
+	AN(l < sizeof buf);
+	s = system(buf);
+	if (s != 0)
+		vtc_log(p->vl, 0, "Failed to send signal (exit status: %d)", s);
+}
+
+static inline void
+process_stop(struct process *p)
+{
+
+	process_kill(p, "TERM");
+}
+
+static inline void
+process_terminate(struct process *p)
+{
+
+	process_kill(p, "TERM");
+	sleep(1);
+	if (p->running && p->pid)
+		process_kill(p, "KILL");
+}
+
+/**********************************************************************
+ * Write to a process' stdin
+ */
+
+static void
+process_write(struct process *p, const char *text)
+{
+	int r, len;
+
+	if (!p->running || !p->pid) {
+		vtc_log(p->vl, 0, "Cannot write to a non-running process");
+		return;
+	}
+
+	len = strlen(text);
+	vtc_log(p->vl, 4, "Writing %d bytes", len);
+	r = write(p->fds[1], text, len);
+	if (r < 0)
+		vtc_log(p->vl, 0, "Failed to write: %s (%d)",
+		    strerror(errno), errno);
+}
+
+static void
+process_close(struct process *p)
+{
+
+	if (!p->running || !p->pid) {
+		vtc_log(p->vl, 0, "Cannot close on a non-running process");
+		return;
+	}
+
+	AZ(close(p->fds[1]));
+	p->fds[1] = -1;
+}
+
+/**********************************************************************
+ * Process command dispatch
+ */
+
+void
+cmd_process(CMD_ARGS)
+{
+	struct process *p, *p2;
+
+	(void)priv;
+	(void)cmd;
+	(void)vl;
+
+	if (av == NULL) {
+		/* Reset and free */
+		VTAILQ_FOREACH_SAFE(p, &processes, list, p2) {
+			if (p->running && p->pid)
+				process_terminate(p);
+			VTAILQ_REMOVE(&processes, p, list);
+			process_delete(p);
+		}
+		return;
+	}
+
+	AZ(strcmp(av[0], "process"));
+	av++;
+
+	VTAILQ_FOREACH(p, &processes, list)
+		if (!strcmp(p->name, av[0]))
+			break;
+	if (p == NULL)
+		p = process_new(av[0]);
+	av++;
+
+	for (; *av != NULL; av++) {
+		if (vtc_error)
+			break;
+
+		if (!strcmp(*av, "-start")) {
+			process_start(p);
+			continue;
+		}
+		if (!strcmp(*av, "-wait")) {
+			process_wait(p);
+			continue;
+		}
+		if (!strcmp(*av, "-kill")) {
+			process_kill(p, av[1]);
+			av++;
+			continue;
+		}
+		if (!strcmp(*av, "-stop")) {
+			process_stop(p);
+			continue;
+		}
+		if (!strcmp(*av, "-write")) {
+			process_write(p, av[1]);
+			av++;
+			continue;
+		}
+		if (!strcmp(*av, "-writeln")) {
+			process_write(p, av[1]);
+			process_write(p, "\n");
+			av++;
+			continue;
+		}
+		if (!strcmp(*av, "-close")) {
+			process_close(p);
+			continue;
+		}
+		if (**av == '-')
+			vtc_log(p->vl, 0, "Unknown process argument: %s", *av);
+		REPLACE(p->spec, *av);
+	}
+}
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index a7b504f..c6cb52f 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -432,6 +432,7 @@ varnish_launch(struct varnish *v)
 	} else {
 		vtc_log(v->vl, 3, "PID: %ld", (long)v->pid);
 		macro_def(v->vl, v->name, "pid", "%ld", (long)v->pid);
+		macro_def(v->vl, v->name, "name", "%s", v->workdir);
 	}
 	AZ(close(v->fds[0]));
 	AZ(close(v->fds[3]));



From phk at FreeBSD.org  Thu Jun 11 09:39:09 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jun 2015 11:39:09 +0200
Subject: [master] 48b09e9 Only run ->fini steps for ->init steps which
 succeeded.
Message-ID: <E1Z2yx3-0006nB-Vf@project.varnish-software.com>


commit 48b09e9d8a90606c45c237ec543ab5925e4d87ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 11 09:38:21 2015 +0000

    Only run ->fini steps for ->init steps which succeeded.
    
    Patch from Martin, with additional attention to VGC layout by me.

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index c4f984d..1ec9d72 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -221,7 +221,7 @@ parse_new(struct vcc *tl)
 	bprintf(buf1, ", &vo_%s, \"%s\"", sy1->name, sy1->name);
 	vcc_Eval_Func(tl, s_init, buf1, sy2->name, s_init + strlen(s_init) + 1);
 	ifp = New_IniFin(tl);
-	VSB_printf(ifp->fin, "\t%s(&vo_%s);", s_fini, sy1->name);
+	VSB_printf(ifp->fin, "\t\t%s(&vo_%s);", s_fini, sy1->name);
 	ExpectErr(tl, ';');
 
 	bprintf(buf1, ", vo_%s", sy1->name);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index d788d28..aff800a 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -413,7 +413,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 	    "\tVRT_init_vbe(ctx, &%s, &vgc_dir_priv_%s);",
 	    vgcname, vgcname);
 	VSB_printf(ifp->fin,
-	    "\tVRT_fini_vbe(ctx, &%s, &vgc_dir_priv_%s);",
+	    "\t\tVRT_fini_vbe(ctx, &%s,\n\t\t    &vgc_dir_priv_%s);",
 	    vgcname, vgcname);
 	VSB_printf(ifp->event,
 	    "\tVRT_event_vbe(ctx, ev, %s, &vgc_dir_priv_%s);",
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 06f3a3e..735341f 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -309,14 +309,19 @@ EmitInitFini(const struct vcc *tl)
 {
 	struct inifin *p;
 
+	Fh(tl, 0, "\nstatic unsigned vgc_inistep;\n");
+
 	/*
 	 * INIT
 	 */
 	Fc(tl, 0, "\nstatic int\nVGC_Load(VRT_CTX)\n{\n\n");
+	Fc(tl, 0, "\tvgc_inistep = 0;\n\n");
 	VTAILQ_FOREACH(p, &tl->inifin, list) {
 		AZ(VSB_finish(p->ini));
+		assert(p->n > 0);
 		if (VSB_len(p->ini))
 			Fc(tl, 0, "\t/* %u */\n%s\n", p->n, VSB_data(p->ini));
+		Fc(tl, 0, "\tvgc_inistep = %u;\n\n", p->n);
 		VSB_delete(p->ini);
 	}
 
@@ -330,8 +335,12 @@ EmitInitFini(const struct vcc *tl)
 
 	VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) {
 		AZ(VSB_finish(p->fin));
-		if (VSB_len(p->fin))
-			Fc(tl, 0, "\t/* %u */\n%s\n", p->n, VSB_data(p->fin));
+		if (VSB_len(p->fin)) {
+			Fc(tl, 0, "\t/* %u */\n", p->n);
+			Fc(tl, 0, "\tif (vgc_inistep >= %u) {\n", p->n);
+			Fc(tl, 0, "%s\n", VSB_data(p->fin));
+			Fc(tl, 0, "\t}\n\n");
+		}
 		VSB_delete(p->fin);
 	}
 
diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c
index 9e8ef9e..c5ea59e 100644
--- a/lib/libvcc/vcc_utils.c
+++ b/lib/libvcc/vcc_utils.c
@@ -75,7 +75,7 @@ vcc_regexp(struct vcc *tl)
 	VSB_printf(ifp->ini, "\tVRT_re_init(&%s, ",buf);
 	EncToken(ifp->ini, tl->t);
 	VSB_printf(ifp->ini, ");");
-	VSB_printf(ifp->fin, "\tVRT_re_fini(%s);", buf);
+	VSB_printf(ifp->fin, "\t\tVRT_re_fini(%s);", buf);
 	return (p);
 }
 
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 8391c98..9f140db 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -181,8 +181,8 @@ vcc_ParseImport(struct vcc *tl)
 	VSB_printf(ifp->ini, "\t\treturn(1);");
 
 	/* XXX: zero the function pointer structure ?*/
-	VSB_printf(ifp->fin, "\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod));
-	VSB_printf(ifp->fin, "\n\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
+	VSB_printf(ifp->fin, "\t\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod));
+	VSB_printf(ifp->fin, "\n\t\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
 
 	ifp = NULL;
 



From phk at FreeBSD.org  Thu Jun 11 10:08:55 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jun 2015 12:08:55 +0200
Subject: [master] 47028c7 Alloc vmods to bail out of ->init,
 now that we have mechanism for doing the right thing during fini.
Message-ID: <E1Z2zPr-0000ib-5A@project.varnish-software.com>


commit 47028c70f9ffa99a1533cf1d3e1725442fda56ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 11 09:47:58 2015 +0000

    Alloc vmods to bail out of ->init, now that we have mechanism for
    doing the right thing during fini.
    
    Patch by: Martin

diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 9f140db..c88c2ad 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -199,7 +199,8 @@ vcc_ParseImport(struct vcc *tl)
 			if (ifp == NULL)
 				ifp = New_IniFin(tl);
 			VSB_printf(ifp->ini,
-			    "\t%s(&vmod_priv_%.*s, &VCL_conf);",
+			    "\tif (%s(&vmod_priv_%.*s, &VCL_conf))\n"
+			    "\t\treturn(1);",
 			    p, PF(mod));
 		} else {
 			sym = VCC_AddSymbolStr(tl, p, SYM_FUNC);



From nils.goroll at uplex.de  Thu Jun 11 11:52:13 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 11 Jun 2015 13:52:13 +0200
Subject: [master] b79f529 exec from the shell so we see the actual pid for the
 SIGHUP
Message-ID: <E1Z311p-00084A-EQ@project.varnish-software.com>


commit b79f5293996250f98d521924644aabc20170e0dd
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jun 11 13:50:40 2015 +0200

    exec from the shell so we see the actual pid for the SIGHUP
    
    Just a quick fix, it would probably be cleaner to avoid the shell
    in the first place.

diff --git a/bin/varnishtest/tests/u00001.vtc b/bin/varnishtest/tests/u00001.vtc
index 7cf18c2..f362a0c 100644
--- a/bin/varnishtest/tests/u00001.vtc
+++ b/bin/varnishtest/tests/u00001.vtc
@@ -12,7 +12,7 @@ server s1 {
 
 varnish v1 -vcl+backend "" -start
 
-process p1 "${varnishncsa} -n ${v1_name} -w ${tmpdir}/ncsa.log -F %s" -start
+process p1 "exec ${varnishncsa} -n ${v1_name} -w ${tmpdir}/ncsa.log -F %s" -start
 
 # give varnishncsa enough time to open the VSM
 delay 1



From phk at FreeBSD.org  Mon Jun 15 07:39:58 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jun 2015 09:39:58 +0200
Subject: [master] 202417a Don't put the cli in VRT_CTX,
 but give it a vsb for complaints during initialization.
Message-ID: <E1Z4Ozu-00039d-Lz@project.varnish-software.com>


commit 202417a5b9cf0da780eca710c4f057aaf5cd166a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 15 07:39:14 2015 +0000

    Don't put the cli in VRT_CTX, but give it a vsb for complaints
    during initialization.
    
    Use vmod_debug to test this.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 18631f8..422d6fd 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -197,6 +197,8 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 	struct VCL_conf const *cnf;
 	struct vrt_ctx ctx;
 	unsigned hand = 0;
+	struct vsb *vsb;
+	int i;
 
 	ASSERT_CLI();
 
@@ -237,26 +239,39 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 	INIT_OBJ(&ctx, VRT_CTX_MAGIC);
 	ctx.method = VCL_MET_INIT;
 	ctx.handling = &hand;
-	ctx.cli = cli;
 	ctx.vcl = vcl->conf;
 
-	if (vcl->conf->event_vcl(&ctx, VCL_EVENT_LOAD)) {
+	vsb = VSB_new_auto();
+	AN(vsb);
+	ctx.msg = vsb;
+	i = vcl->conf->event_vcl(&ctx, VCL_EVENT_LOAD);
+	AZ(VSB_finish(vsb));
+	if (i) {
 		VCLI_Out(cli, "VCL \"%s\" Failed to initialize", name);
+		if (VSB_len(vsb))
+			VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(vsb));
 		AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
 		(void)dlclose(vcl->dlh);
 		FREE_OBJ(vcl);
+		VSB_delete(vsb);
 		return (1);
 	}
+	VSB_clear(vsb);
 	(void)vcl->conf->init_func(&ctx);
+	AZ(VSB_finish(vsb));
 	if (hand == VCL_RET_FAIL) {
 		VCLI_Out(cli, "VCL \"%s\" vcl_init{} failed", name);
+		if (VSB_len(vsb))
+			VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(vsb));
 		ctx.method = VCL_MET_FINI;
 		(void)vcl->conf->fini_func(&ctx);
 		AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
 		(void)dlclose(vcl->dlh);
 		FREE_OBJ(vcl);
+		VSB_delete(vsb);
 		return (1);
 	}
+	VSB_delete(vsb);
 	vcl_set_state(vcl, state);
 	assert(hand == VCL_RET_OK);
 	VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name);
@@ -392,6 +407,8 @@ ccf_config_use(struct cli *cli, const char * const *av, void *priv)
 	struct vcls *vcl;
 	struct vrt_ctx ctx;
 	unsigned hand = 0;
+	struct vsb *vsb;
+	int i;
 
 	ASSERT_CLI();
 	AZ(priv);
@@ -400,16 +417,23 @@ ccf_config_use(struct cli *cli, const char * const *av, void *priv)
 	AN(vcl->warm);			// MGT ensures this
 	INIT_OBJ(&ctx, VRT_CTX_MAGIC);
 	ctx.handling = &hand;
-	ctx.cli = cli;
-	if (vcl->conf->event_vcl(&ctx, VCL_EVENT_USE)) {
+	vsb = VSB_new_auto();
+	AN(vsb);
+	ctx.msg = vsb;
+	i = vcl->conf->event_vcl(&ctx, VCL_EVENT_USE);
+	AZ(VSB_finish(vsb));
+	if (i) {
 		VCLI_Out(cli, "VCL \"%s\" Failed to activate", av[2]);
+		if (VSB_len(vsb) > 0)
+			VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(vsb));
 		VCLI_SetResult(cli, CLIS_CANT);
-		return;
+	} else {
+		Lck_Lock(&vcl_mtx);
+		vcl_active = vcl;
+		Lck_Unlock(&vcl_mtx);
 	}
-
-	Lck_Lock(&vcl_mtx);
-	vcl_active = vcl;
-	Lck_Unlock(&vcl_mtx);
+	VSB_delete(vsb);
+	return;
 }
 
 static void __match_proto__(cli_func_t)
diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c
index 5245708..50e2d61 100644
--- a/bin/varnishd/cache/cache_vrt_vmod.c
+++ b/bin/varnishd/cache/cache_vrt_vmod.c
@@ -72,15 +72,15 @@ VRT_Vmod_Init(struct vmod **hdl, void *ptr, int len, const char *nm,
 
 	ASSERT_CLI();
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	AN(ctx->cli);
+	AN(ctx->msg);
 	AN(hdl);
 	AZ(*hdl);
 
 	dlhdl = dlopen(path, RTLD_NOW | RTLD_LOCAL);
 	if (dlhdl == NULL) {
-		VCLI_Out(ctx->cli, "Loading VMOD %s from %s:\n", nm, path);
-		VCLI_Out(ctx->cli, "dlopen() failed: %s\n", dlerror());
-		VCLI_Out(ctx->cli, "Check child process permissions.\n");
+		VSB_printf(ctx->msg, "Loading VMOD %s from %s:\n", nm, path);
+		VSB_printf(ctx->msg, "dlopen() failed: %s\n", dlerror());
+		VSB_printf(ctx->msg, "Check child process permissions.\n");
 		return (1);
 	}
 
@@ -98,9 +98,9 @@ VRT_Vmod_Init(struct vmod **hdl, void *ptr, int len, const char *nm,
 		if (d == NULL ||
 		    d->file_id == NULL ||
 		    strcmp(d->file_id, file_id)) {
-			VCLI_Out(ctx->cli,
+			VSB_printf(ctx->msg,
 			    "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(ctx->cli,
+			VSB_printf(ctx->msg,
 			    "This is no longer the same file seen by"
 			    " the VCL-compiler.\n");
 			(void)dlclose(v->hdl);
@@ -116,9 +116,9 @@ VRT_Vmod_Init(struct vmod **hdl, void *ptr, int len, const char *nm,
 		    d->proto == NULL ||
 		    d->spec == NULL ||
 		    d->abi == NULL) {
-			VCLI_Out(ctx->cli,
+			VSB_printf(ctx->msg,
 			    "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(ctx->cli, "VMOD data is mangled.\n");
+			VSB_printf(ctx->msg, "VMOD data is mangled.\n");
 			(void)dlclose(v->hdl);
 			FREE_OBJ(v);
 			return (1);
diff --git a/bin/varnishtest/tests/m00022.vtc b/bin/varnishtest/tests/m00022.vtc
new file mode 100644
index 0000000..5febb43
--- /dev/null
+++ b/bin/varnishtest/tests/m00022.vtc
@@ -0,0 +1,28 @@
+varnishtest "Test std & debug vmod"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend { } -start
+
+varnish v1 -errvcl "Planned failure in vcl_init" {
+
+	import ${vmod_debug};
+
+	backend default {
+		.host = "${s1_addr}";
+	}
+
+	sub vcl_init {
+		debug.init_fail();
+		return (fail);
+	}
+}
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/include/vrt.h b/include/vrt.h
index dca5902..4ccadbc 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -54,7 +54,7 @@ struct busyobj;
 struct vsl_log;
 struct http;
 struct ws;
-struct cli;
+struct vsb;
 struct director;
 struct VCL_conf;
 struct suckaddr;
@@ -93,7 +93,7 @@ struct vrt_ctx {
 	unsigned			method;
 	unsigned			*handling;
 
-	struct cli			*cli;	// Only in ...init()
+	struct vsb			*msg;	// Only in ...init()
 	struct vsl_log			*vsl;
 	struct VCL_conf			*vcl;
 	struct ws			*ws;
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 434ee78..2e08154 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -101,3 +101,7 @@ $Function INT vre_limit()
 $Function VOID register_exp_callback(PRIV_VCL)
 
 Register the vmod to receive expiry callbacks
+
+$Function VOID init_fail()
+
+Function to fail vcl_init{}
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index ccdcab3..9e02297 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -34,6 +34,7 @@
 #include "cache/cache.h"
 
 #include "vrt.h"
+#include "vsb.h"
 #include "vcc_if.h"
 
 struct priv_vcl {
@@ -220,6 +221,14 @@ vmod_register_exp_callback(VRT_CTX, struct vmod_priv *priv)
 	VSL(SLT_Debug, 0, "exp_cb: registered");
 }
 
+VCL_VOID __match_proto__()
+vmod_init_fail(VRT_CTX)
+{
+
+	AN(ctx->msg);
+	VSB_printf(ctx->msg, "Planned failure in vcl_init{}");
+}
+
 static void __match_proto__(vmod_priv_free_f)
 priv_vcl_free(void *priv)
 {



From phk at FreeBSD.org  Mon Jun 15 07:55:45 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jun 2015 09:55:45 +0200
Subject: [master] 80beeb4 Make it possible for (vmod) functions called from
 vcl_init{} to set "return(fail)" status, by making the "return(ok)" implicit.
Message-ID: <E1Z4PFB-0004va-UM@project.varnish-software.com>


commit 80beeb4104313040bcb19c9e805e8431c4d12ef3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 15 07:54:38 2015 +0000

    Make it possible for (vmod) functions called from vcl_init{} to set
    "return(fail)" status, by making the "return(ok)" implicit.
    
    Notice that setting a fail status does not break off vcl_init{} execution,
    and that the final status is simply the last status set.

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 3ebe9ce..e4c2a6f 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -192,7 +192,6 @@ sub vcl_backend_error {
 # Housekeeping
 
 sub vcl_init {
-    return (ok);
 }
 
 sub vcl_fini {
diff --git a/bin/varnishtest/tests/m00022.vtc b/bin/varnishtest/tests/m00022.vtc
index 5febb43..c1d3542 100644
--- a/bin/varnishtest/tests/m00022.vtc
+++ b/bin/varnishtest/tests/m00022.vtc
@@ -1,4 +1,4 @@
-varnishtest "Test std & debug vmod"
+varnishtest "Test vmod failure in vcl_init{}"
 
 server s1 {
 	rxreq
@@ -17,7 +17,6 @@ varnish v1 -errvcl "Planned failure in vcl_init" {
 
 	sub vcl_init {
 		debug.init_fail();
-		return (fail);
 	}
 }
 
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 735341f..e837742 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -700,7 +700,16 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 		    method_tab[i].name);
 		AZ(VSB_finish(tl->fm[i]));
 		Fc(tl, 1, "{\n");
+		/*
+		 * We want vmods to be able set a FAIL return value
+		 * in members called from vcl_init, so set OK up front
+		 * and return with whatever was set last.
+		 */
+		if (method_tab[i].bitval == VCL_MET_INIT)
+			Fc(tl, 1, "  VRT_handling(ctx, VCL_RET_OK);\n");
 		Fc(tl, 1, "%s", VSB_data(tl->fm[i]));
+		if (method_tab[i].bitval == VCL_MET_INIT)
+			Fc(tl, 1, "  return(1);\n");
 		Fc(tl, 1, "}\n");
 	}
 
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 9e02297..28f37ab 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -33,6 +33,7 @@
 
 #include "cache/cache.h"
 
+#include "vcl.h"
 #include "vrt.h"
 #include "vsb.h"
 #include "vcc_if.h"
@@ -227,6 +228,7 @@ vmod_init_fail(VRT_CTX)
 
 	AN(ctx->msg);
 	VSB_printf(ctx->msg, "Planned failure in vcl_init{}");
+	VRT_handling(ctx, VCL_RET_FAIL);
 }
 
 static void __match_proto__(vmod_priv_free_f)



From phk at FreeBSD.org  Mon Jun 15 08:26:00 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jun 2015 10:26:00 +0200
Subject: [master] 25ae15e Call vcl_init{} from the compiled VMODs event(LOAD)
 code.
Message-ID: <E1Z4PiS-00085O-6Q@project.varnish-software.com>


commit 25ae15ef46c04c0a3e81a264ccd627873d9482a2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 15 08:25:30 2015 +0000

    Call vcl_init{} from the compiled VMODs event(LOAD) code.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 422d6fd..ff25f62 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -247,7 +247,7 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 	i = vcl->conf->event_vcl(&ctx, VCL_EVENT_LOAD);
 	AZ(VSB_finish(vsb));
 	if (i) {
-		VCLI_Out(cli, "VCL \"%s\" Failed to initialize", name);
+		VCLI_Out(cli, "VCL \"%s\" Failed initialization", name);
 		if (VSB_len(vsb))
 			VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(vsb));
 		AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
@@ -256,21 +256,6 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 		VSB_delete(vsb);
 		return (1);
 	}
-	VSB_clear(vsb);
-	(void)vcl->conf->init_func(&ctx);
-	AZ(VSB_finish(vsb));
-	if (hand == VCL_RET_FAIL) {
-		VCLI_Out(cli, "VCL \"%s\" vcl_init{} failed", name);
-		if (VSB_len(vsb))
-			VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(vsb));
-		ctx.method = VCL_MET_FINI;
-		(void)vcl->conf->fini_func(&ctx);
-		AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
-		(void)dlclose(vcl->dlh);
-		FREE_OBJ(vcl);
-		VSB_delete(vsb);
-		return (1);
-	}
 	VSB_delete(vsb);
 	vcl_set_state(vcl, state);
 	assert(hand == VCL_RET_OK);
diff --git a/bin/varnishtest/tests/v00040.vtc b/bin/varnishtest/tests/v00040.vtc
index a21ca4b..408169f 100644
--- a/bin/varnishtest/tests/v00040.vtc
+++ b/bin/varnishtest/tests/v00040.vtc
@@ -12,7 +12,7 @@ client c1 {
 	rxresp
 } -run
 
-varnish v1 -errvcl {VCL "vcl2" vcl_init{} failed} {
+varnish v1 -errvcl {VCL "vcl2" Failed initialization} {
 	sub vcl_init {
 		return (fail);
 	}
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index aff800a..59fb851 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -410,7 +410,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 
 	ifp = New_IniFin(tl);
 	VSB_printf(ifp->ini,
-	    "\tVRT_init_vbe(ctx, &%s, &vgc_dir_priv_%s);",
+	    "\tVRT_init_vbe(ctx, &%s,\n\t    &vgc_dir_priv_%s);",
 	    vgcname, vgcname);
 	VSB_printf(ifp->fin,
 	    "\t\tVRT_fini_vbe(ctx, &%s,\n\t\t    &vgc_dir_priv_%s);",
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index e837742..24ca4df 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -325,7 +325,8 @@ EmitInitFini(const struct vcc *tl)
 		VSB_delete(p->ini);
 	}
 
-	Fc(tl, 0, "\treturn(0);\n");
+	Fc(tl, 0, "\t(void)VGC_function_vcl_init(ctx);\n");
+	Fc(tl, 0, "\treturn(*ctx->handling == VCL_RET_OK ? 0: -1);\n");
 	Fc(tl, 0, "}\n");
 
 	/*



From phk at FreeBSD.org  Mon Jun 15 09:02:18 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jun 2015 11:02:18 +0200
Subject: [master] f6a324e Pass the CTX to vmods init functions instad of the
 VCL_conf (which is part of the CTX).
Message-ID: <E1Z4QHa-0003NN-KL@project.varnish-software.com>


commit f6a324ebd084332589a6a2bfc8edbd5c2c9c48c2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 15 09:01:35 2015 +0000

    Pass the CTX to vmods init functions instad of the VCL_conf (which is
    part of the CTX).
    
    Test that failing vmods init functions works.
    
    Based on patch from Martin

diff --git a/bin/varnishtest/tests/m00022.vtc b/bin/varnishtest/tests/m00022.vtc
index c1d3542..29da34e 100644
--- a/bin/varnishtest/tests/m00022.vtc
+++ b/bin/varnishtest/tests/m00022.vtc
@@ -20,6 +20,17 @@ varnish v1 -errvcl "Planned failure in vcl_init" {
 	}
 }
 
+varnish v1 -cliok "param.set nuke_limit 42"
+
+varnish v1 -errvcl "nuke_limit is not the answer." {
+
+	import ${vmod_debug};
+
+	backend default {
+		.host = "${s1_addr}";
+	}
+}
+
 client c1 {
 	txreq
 	rxresp
diff --git a/include/vrt.h b/include/vrt.h
index 4ccadbc..36b0165 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -42,9 +42,9 @@
  * - cache_vrt.c: -> ssize_t VRT_CacheReqBody(VRT_CTX, size_t)
  */
 
-#define VRT_MAJOR_VERSION	2U
+#define VRT_MAJOR_VERSION	3U
 
-#define VRT_MINOR_VERSION	2U
+#define VRT_MINOR_VERSION	0U
 
 
 /***********************************************************************/
@@ -265,7 +265,7 @@ struct vmod_priv {
 	vmod_priv_free_f	*free;
 };
 
-typedef int vmod_init_f(struct vmod_priv *,  const struct VCL_conf *);
+typedef int vmod_init_f(VRT_CTX, struct vmod_priv *);
 
 void VRT_priv_fini(const struct vmod_priv *p);
 struct vmod_priv *VRT_priv_task(VRT_CTX, void *vmod_id);
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index c88c2ad..cbae127 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -199,7 +199,7 @@ vcc_ParseImport(struct vcc *tl)
 			if (ifp == NULL)
 				ifp = New_IniFin(tl);
 			VSB_printf(ifp->ini,
-			    "\tif (%s(&vmod_priv_%.*s, &VCL_conf))\n"
+			    "\tif (%s(ctx, &vmod_priv_%.*s))\n"
 			    "\t\treturn(1);",
 			    p, PF(mod));
 		} else {
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index bc4c3c4..4619f1d 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -189,7 +189,7 @@ class Vmod(object):
 			fo.write("\n")
 			fo.write("int " + self.init)
 			fo.write(
-			    "(struct vmod_priv *, const struct VCL_conf *);\n")
+			    "(VRT_CTX, struct vmod_priv *);\n")
 
 	def c_typedefs_(self):
 		l = list()
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 28f37ab..a91e845 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -248,11 +248,16 @@ priv_vcl_free(void *priv)
 }
 
 int __match_proto__(vmod_init_f)
-init_function(struct vmod_priv *priv, const struct VCL_conf *cfg)
+init_function(VRT_CTX, struct vmod_priv *priv)
 {
 	struct priv_vcl *priv_vcl;
 
-	(void)cfg;
+	AN(ctx->msg);
+
+	if (cache_param->nuke_limit == 42) {
+		VSB_printf(ctx->msg, "nuke_limit is not the answer.");
+		return (-1);
+	}
 
 	ALLOC_OBJ(priv_vcl, PRIV_VCL_MAGIC);
 	AN(priv_vcl);



From phk at FreeBSD.org  Mon Jun 15 10:40:05 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jun 2015 12:40:05 +0200
Subject: [master] 968fc3e Call the vcl_fini{} from the VGC's event(DISCARD).
Message-ID: <E1Z4RoD-0003uG-6k@project.varnish-software.com>


commit 968fc3ecf4293b27c195bbb6e9d0605c9195973c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 15 10:39:06 2015 +0000

    Call the vcl_fini{} from the VGC's event(DISCARD).

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index ff25f62..2a98fd8 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -290,8 +290,6 @@ VCL_Nuke(struct vcls *vcl)
 	ctx.method = VCL_MET_FINI;
 	ctx.handling = &hand;
 	ctx.vcl = vcl->conf;
-	(void)vcl->conf->fini_func(&ctx);
-	assert(hand == VCL_RET_OK);
 	AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
 	free(vcl->conf->loaded_name);
 	(void)dlclose(vcl->dlh);
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 24ca4df..21fc395 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -334,6 +334,7 @@ EmitInitFini(const struct vcc *tl)
 	 */
 	Fc(tl, 0, "\nstatic int\nVGC_Discard(VRT_CTX)\n{\n\n");
 
+	Fc(tl, 0, "\t(void)VGC_function_vcl_fini(ctx);\n\n");
 	VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) {
 		AZ(VSB_finish(p->fin));
 		if (VSB_len(p->fin)) {



From fgsch at lodoss.net  Mon Jun 15 14:52:05 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 15 Jun 2015 16:52:05 +0200
Subject: [master] a7f8e65 Improve description
Message-ID: <E1Z4Vk5-0004Jp-47@project.varnish-software.com>


commit a7f8e65880025f8ef6e93779cf13767a9bd5ae58
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 15 15:50:14 2015 +0100

    Improve description

diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 82bf0aa..91fbe30 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -86,9 +86,9 @@ Example
 $Function VOID syslog(INT priority, STRING_LIST s)
 
 Description
-	Logs the string *s* to syslog marked with *priority*. *priority*
-	is formed by ORing the facility and priority values.  See your
-	system's syslog.h file for priorities and facility codes.
+	Logs the string *s* to syslog tagged with *priority*. *priority*
+	is formed by ORing the facility and level values. See your
+	system's syslog.h file for possible values.
 Example
 	std.syslog(9, "Something is wrong");
 
@@ -107,13 +107,14 @@ Example
 $Function VOID collect(HEADER hdr)
 
 Description
-	Collapses the header *hdr*, joining them into one.
+	Collapses multiple *hdr* headers into one long header.
+
+	Care should be taken when collapsing headers.  In particular
+	collapsing Set-Cookie will lead to unexpected results on the
+	browser side.
 Example
 	std.collect(req.http.cookie);
 
-	This will collapse several Cookie: headers into one, long
-	cookie header.
-
 $Function DURATION duration(STRING s, DURATION fallback)
 
 Description



From fgsch at lodoss.net  Mon Jun 15 15:32:58 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 15 Jun 2015 17:32:58 +0200
Subject: [master] 2301a32 Drop VCL_conf forward declaration
Message-ID: <E1Z4WNe-0000PE-Is@project.varnish-software.com>


commit 2301a32f487b632ba6a7c770fcef154a5431fe2b
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 15 16:27:52 2015 +0100

    Drop VCL_conf forward declaration
    
    No longer needed after f6a324ebd084332589a6a2bfc8edbd5c2c9c48c2.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 4619f1d..c2c16c5 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -897,7 +897,6 @@ def runmain(inputvcc, outputname="vcc_if"):
 	write_c_file_warning(fc)
 	write_c_file_warning(fh)
 
-	fh.write('struct VCL_conf;\n')
 	fh.write('struct vmod_priv;\n')
 	fh.write("\n")
 



From fgsch at lodoss.net  Mon Jun 15 15:32:58 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 15 Jun 2015 17:32:58 +0200
Subject: [master] 74fba61 Document BLOB and improve existing bits
Message-ID: <E1Z4WNe-0000PK-KT@project.varnish-software.com>


commit 74fba61a236673450729b11133a3fac59e84e6b3
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 15 16:29:53 2015 +0100

    Document BLOB and improve existing bits

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 14bb027..c339f0e 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -75,7 +75,7 @@ primary action, something functions which return a value can not::
 		std.set_ip_tos(32);
 	}
 
-Running vmodtool.py on the vmod.vcc file, produces an "vcc_if.c" and
+Running vmodtool.py on the vmod.vcc file, produces a "vcc_if.c" and
 "vcc_if.h" files, which you must use to build your shared library
 file.
 
@@ -88,15 +88,13 @@ the functions you want to export to VCL.
 
 For the std VMOD, the compiled vcc_if.h file looks like this::
 
-	struct vrt_ctx;
-	struct VCL_conf;
 	struct vmod_priv;
 
-	VCL_STRING vmod_toupper(const struct vrt_ctx *, const char *, ...);
-	VCL_STRING vmod_tolower(const struct vrt_ctx *, const char *, ...);
-	VCL_VOID vmod_set_ip_tos(const struct vrt_ctx *, VCL_INT);
+	VCL_STRING vmod_toupper(VRT_CTX, const char *, ...);
+	VCL_STRING vmod_tolower(VRT_CTX, const char *, ...);
+	VCL_VOID vmod_set_ip_tos(VRT_CTX, VCL_INT);
 
-	int init_function(struct vmod_priv *, const struct VCL_conf *);
+	int init_function(VRT_CTX, struct vmod_priv *);
 
 Those are your C prototypes.  Notice the ``vmod_`` prefix on the function
 names and the C-types as arguments.
@@ -108,31 +106,32 @@ VCL data types are targeted at the job, so for instance, we have data
 types like "DURATION" and "HEADER", but they all have some kind of C
 language representation.  Here is a description of them.
 
-All but the STRING_LIST type have typedefs:  VCL_INT, VCL_REAL etc.
+All but the STRING_LIST type have typedefs: VCL_INT, VCL_REAL etc.
 
 INT
-	C-type: ``int``
+	C-type: ``long``
 
 	An integer as we know and love them.
 
 REAL
 	C-type: ``double``
 
-	A floating point value
+	A floating point value.
 
 DURATION
 	C-type: ``double``
 
-	Units: seconds
+	Unit: seconds
 
-	A time interval, as in "25 minutes".
+	A time interval, as in 25 seconds.
 
 TIME
 	C-type: ``double``
 
-	Units: seconds since UNIX epoch
+	Unit: seconds since UNIX epoch
 
-	An absolute time, as in "Mon Sep 13 19:06:01 UTC 2010".
+	An absolute time, as in 1284401161.  When used in a string
+	context is formatted as "Mon, 13 Sep 2010 19:06:01 GMT".
 
 STRING
 	C-type: ``const char *``
@@ -229,6 +228,12 @@ BOOL
 
 	Zero means false, anything else means true.
 
+BLOB
+	C-type: ``const struct vmod_priv *``
+
+	An opaque type to pass random bits of memory between VMOD
+	functions.
+
 
 Private Pointers
 ================
@@ -266,17 +271,20 @@ The way it works in the vmod code, is that a ``struct vmod_priv *`` is
 passed to the functions where one of the ``PRIV_*`` argument types is
 specified.
 
-This structure contains two members::
+This structure contains three members::
 
 	typedef void vmod_priv_free_f(void *);
 	struct vmod_priv {
 		void                    *priv;
+		int			len;
 		vmod_priv_free_f        *free;
 	};
 
 The "priv" element can be used for whatever the vmod code wants to
 use it for, it defaults to a NULL pointer.
 
+The "len" element is used primarily for BLOBs to indicate its size.
+
 The "free" element defaults to NULL, and it is the modules responsibility
 to set it to a suitable function, which can clean up whatever the "priv"
 pointer points to.



From phk at FreeBSD.org  Tue Jun 16 09:58:10 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jun 2015 11:58:10 +0200
Subject: [master] 5dd668e Make the test and real opens of compiled VCL's use
 the same code.
Message-ID: <E1Z4ndC-0008Vx-RA@project.varnish-software.com>


commit 5dd668e39d99f9ae217061e20fb58d9b2010238f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 16 09:57:23 2015 +0000

    Make the test and real opens of compiled VCL's use the same code.
    
    Move the meat of the 'test-load' to cache_vcl.c, so that vcl.h can
    become a worker-only include file.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 2a98fd8..046b731 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -155,6 +155,79 @@ VCL_Rel(struct VCL_conf **vcc)
 /*--------------------------------------------------------------------*/
 
 static struct vcls *
+VCL_Open(const char *fn, struct vsb *msg)
+{
+	struct vcls *vcl;
+	void *dlh;
+	struct VCL_conf const *cnf;
+
+	AN(fn);
+	AN(msg);
+
+	dlh = dlopen(fn, RTLD_NOW | RTLD_LOCAL);
+	if (dlh == NULL) {
+		VSB_printf(msg, "Could not load compiled VCL.\n");
+		VSB_printf(msg, "\tdlopen(%s) = %s\n", fn, dlerror());
+		return (NULL);
+	}
+	cnf = dlsym(dlh, "VCL_conf");
+	if (cnf == NULL) {
+		VSB_printf(msg, "Compiled VCL lacks metadata.\n");
+		(void)dlclose(dlh);
+		return (NULL);
+	}
+	if (cnf->magic != VCL_CONF_MAGIC) {
+		VSB_printf(msg, "Compiled VCL has mangled metadata.\n");
+		(void)dlclose(dlh);
+		return (NULL);
+	}
+	ALLOC_OBJ(vcl, VVCLS_MAGIC);
+	AN(vcl);
+	vcl->dlh = dlh;
+	memcpy(vcl->conf, cnf, sizeof *cnf);
+	return (vcl);
+}
+
+static void
+VCL_Close(struct vcls **vclp)
+{
+	struct vcls *vcl;
+
+	CHECK_OBJ_NOTNULL(*vclp, VVCLS_MAGIC);
+	vcl = *vclp;
+	*vclp = NULL;
+	AZ(dlclose(vcl->dlh));
+	FREE_OBJ(vcl);
+}
+
+/*--------------------------------------------------------------------
+ * NB: This function is called from the test-load subprocess.
+ */
+
+int
+VCL_TestLoad(const char *fn)
+{
+	struct vsb *vsb;
+	struct vcls *vcl;
+	int retval = 0;
+
+	AN(fn);
+	vsb = VSB_new_auto();
+	AN(vsb);
+	vcl = VCL_Open(fn, vsb);
+	if (vcl == NULL) {
+		AZ(VSB_finish(vsb));
+		fprintf(stderr, "%s", VSB_data(vsb));
+		retval = -1;
+	} else
+		VCL_Close(&vcl);
+	VSB_delete(vsb);
+	return (retval);
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct vcls *
 vcl_find(const char *name)
 {
 	struct vcls *vcl;
@@ -194,7 +267,6 @@ static int
 VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 {
 	struct vcls *vcl;
-	struct VCL_conf const *cnf;
 	struct vrt_ctx ctx;
 	unsigned hand = 0;
 	struct vsb *vsb;
@@ -208,41 +280,26 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 		return (1);
 	}
 
-	ALLOC_OBJ(vcl, VVCLS_MAGIC);
-	AN(vcl);
-
-	vcl->dlh = dlopen(fn, RTLD_NOW | RTLD_LOCAL);
+	vsb = VSB_new_auto();
+	AN(vsb);
 
-	if (vcl->dlh == NULL) {
-		VCLI_Out(cli, "dlopen(%s): %s\n", fn, dlerror());
-		FREE_OBJ(vcl);
-		return (1);
-	}
-	cnf = dlsym(vcl->dlh, "VCL_conf");
-	if (cnf == NULL) {
-		VCLI_Out(cli, "Internal error: No VCL_conf symbol\n");
-		(void)dlclose(vcl->dlh);
-		FREE_OBJ(vcl);
+	vcl = VCL_Open(fn, vsb);
+	if (vcl == NULL) {
+		AZ(VSB_finish(vsb));
+		VCLI_Out(cli, "%s", VSB_data(vsb));
+		VSB_delete(vsb);
 		return (1);
 	}
-	memcpy(vcl->conf, cnf, sizeof *cnf);
+
 	vcl->conf->loaded_name = strdup(name);
 	XXXAN(vcl->conf->loaded_name);
 
-	if (vcl->conf->magic != VCL_CONF_MAGIC) {
-		VCLI_Out(cli, "Wrong VCL_CONF_MAGIC\n");
-		(void)dlclose(vcl->dlh);
-		FREE_OBJ(vcl);
-		return (1);
-	}
-
 	INIT_OBJ(&ctx, VRT_CTX_MAGIC);
 	ctx.method = VCL_MET_INIT;
 	ctx.handling = &hand;
 	ctx.vcl = vcl->conf;
 
-	vsb = VSB_new_auto();
-	AN(vsb);
+	VSB_clear(vsb);
 	ctx.msg = vsb;
 	i = vcl->conf->event_vcl(&ctx, VCL_EVENT_LOAD);
 	AZ(VSB_finish(vsb));
@@ -251,8 +308,7 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 		if (VSB_len(vsb))
 			VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(vsb));
 		AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
-		(void)dlclose(vcl->dlh);
-		FREE_OBJ(vcl);
+		VCL_Close(&vcl);
 		VSB_delete(vsb);
 		return (1);
 	}
@@ -292,8 +348,7 @@ VCL_Nuke(struct vcls *vcl)
 	ctx.vcl = vcl->conf;
 	AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
 	free(vcl->conf->loaded_name);
-	(void)dlclose(vcl->dlh);
-	FREE_OBJ(vcl);
+	VCL_Close(&vcl);
 	VSC_C_main->n_vcl--;
 	VSC_C_main->n_vcl_discard--;
 }
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index f487886..a97986c 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -105,6 +105,8 @@ void mgt_child_inherit(int fd, const char *what);
 
 #define NEEDLESS_RETURN(foo)	return (foo)
 
+/* cache/cache_vcl.c */
+int VCL_TestLoad(const char *);
 
 /* vsm.c */
 struct vsm_sc;
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index 4ee4fc7..8a72f20 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -44,7 +44,6 @@
 #include "mgt/mgt.h"
 
 #include "libvcc.h"
-#include "vcl.h"
 #include "vcli.h"
 #include "vcli_priv.h"
 #include "vfil.h"
@@ -184,37 +183,12 @@ run_cc(void *priv)
 static void __match_proto__(vsub_func_f)
 run_dlopen(void *priv)
 {
-	void *dlh;
-	struct VCL_conf const *cnf;
 	struct vcc_priv *vp;
 
 	VJ_subproc(JAIL_SUBPROC_VCLLOAD);
 	CAST_OBJ_NOTNULL(vp, priv, VCC_PRIV_MAGIC);
-
-	/* Try to load the object into this sub-process */
-	if ((dlh = dlopen(vp->libfile, RTLD_NOW | RTLD_LOCAL)) == NULL) {
-		fprintf(stderr, "Compiled VCL program failed to load:\n  %s\n",
-		    dlerror());
-		exit(1);
-	}
-
-	cnf = dlsym(dlh, "VCL_conf");
-	if (cnf == NULL) {
-		fprintf(stderr, "Compiled VCL program, metadata not found\n");
+	if (VCL_TestLoad(vp->libfile))
 		exit(1);
-	}
-
-	if (cnf->magic != VCL_CONF_MAGIC) {
-		fprintf(stderr, "Compiled VCL program, mangled metadata\n");
-		exit(1);
-	}
-
-	if (dlclose(dlh)) {
-		fprintf(stderr,
-		    "Compiled VCL program failed to unload:\n  %s\n",
-		    dlerror());
-		exit(1);
-	}
 	exit(0);
 }
 
diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c
index 278dd9d..7971346 100644
--- a/bin/varnishd/mgt/mgt_vcl.c
+++ b/bin/varnishd/mgt/mgt_vcl.c
@@ -41,7 +41,6 @@
 #include "mgt/mgt.h"
 #include "common/params.h"
 
-#include "vcl.h"
 #include "vcli.h"
 #include "vcli_priv.h"
 #include "vev.h"



From phk at FreeBSD.org  Tue Jun 16 10:57:05 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jun 2015 12:57:05 +0200
Subject: [master] 4ce9f90 Turn the optional vmod init function into an event
 function instead, and call it with LOAD and DISCARD events.
Message-ID: <E1Z4oYD-0005Il-Ot@project.varnish-software.com>


commit 4ce9f900a1ce725735a48002e6dda437fe9a4fde
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 16 10:56:34 2015 +0000

    Turn the optional vmod init function into an event function instead,
    and call it with LOAD and DISCARD events.

diff --git a/include/vrt.h b/include/vrt.h
index 36b0165..cbed87b 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -265,7 +265,9 @@ struct vmod_priv {
 	vmod_priv_free_f	*free;
 };
 
-typedef int vmod_init_f(VRT_CTX, struct vmod_priv *);
+#ifdef VCL_RET_MAX
+typedef int vmod_event_f(VRT_CTX, struct vmod_priv *, enum vcl_event_e);
+#endif
 
 void VRT_priv_fini(const struct vmod_priv *p);
 struct vmod_priv *VRT_priv_task(VRT_CTX, void *vmod_id);
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index cbae127..ffb70b7 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -189,19 +189,22 @@ vcc_ParseImport(struct vcc *tl)
 	spec = vmd->spec;
 	for (; *spec != NULL; spec++) {
 		p = *spec;
-		if (!strcmp(p, "OBJ")) {
+		if (!strcmp(p, "$OBJ")) {
 			p += strlen(p) + 1;
 			sym = VCC_AddSymbolStr(tl, p, SYM_OBJECT);
 			XXXAN(sym);
 			sym->args = p;
-		} else if (!strcmp(p, "INIT")) {
+		} else if (!strcmp(p, "$EVENT")) {
 			p += strlen(p) + 1;
 			if (ifp == NULL)
 				ifp = New_IniFin(tl);
 			VSB_printf(ifp->ini,
-			    "\tif (%s(ctx, &vmod_priv_%.*s))\n"
+			    "\tif (%s(ctx, &vmod_priv_%.*s, VCL_EVENT_LOAD))\n"
 			    "\t\treturn(1);",
 			    p, PF(mod));
+			VSB_printf(ifp->fin,
+			    "\t\t(void)%s(ctx, &vmod_priv_%.*s,\n"
+			    "\t\t    VCL_EVENT_DISCARD);\n", p, PF(mod));
 		} else {
 			sym = VCC_AddSymbolStr(tl, p, SYM_FUNC);
 			ERRCHK(tl);
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index c2c16c5..a5eb8bb 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -153,18 +153,19 @@ class Vmod(object):
 		self.nam = nam
 		self.dnam = dnam
 		self.sec = sec
-		self.init = None
+		self.event = None
 		self.funcs = list()
 		self.objs = list()
 		self.doc_str = []
 		self.doc_order = []
 
-	def set_init(self, nam):
-		if self.init != None:
-			raise ParseError("Module %s already has Init", self.nam)
+	def set_event(self, nam):
+		if self.event != None:
+			raise ParseError("Module %s already has $Event",
+			    self.nam)
 		if not is_c_name(nam):
-			raise ParseError("Init name '%s' is illegal", nam)
-		self.init = nam
+			raise ParseError("$Event name '%s' is illegal", nam)
+		self.event = nam
 
 	def add_func(self, fn):
 		self.funcs.append(fn)
@@ -185,11 +186,11 @@ class Vmod(object):
 		for f in self.funcs:
 			for i in lwrap(f.c_proto()):
 				fo.write(i + "\n")
-		if self.init != None:
+		if self.event != None:
 			fo.write("\n")
-			fo.write("int " + self.init)
-			fo.write(
-			    "(VRT_CTX, struct vmod_priv *);\n")
+			fo.write("#ifdef VCL_MET_MAX\n")
+			fo.write("vmod_event_f " + self.event + ";\n")
+			fo.write("#endif\n")
 
 	def c_typedefs_(self):
 		l = list()
@@ -265,8 +266,8 @@ class Vmod(object):
 			s += f.c_initializer()
 
 		s += "\n\t/* Init/Fini */\n"
-		if self.init != None:
-			s += "\t" + self.init + ",\n"
+		if self.event != None:
+			s += "\t" + self.event + ",\n"
 		s += "};"
 
 		return s
@@ -281,8 +282,8 @@ class Vmod(object):
 			s += f.c_struct(self.nam)
 
 		s += "\n\t/* Init/Fini */\n"
-		if self.init != None:
-			s += "\tvmod_init_f\t*_init;\n"
+		if self.event != None:
+			s += "\tvmod_event_f\t*_event;\n"
 		s += '}'
 		return s
 
@@ -297,9 +298,9 @@ class Vmod(object):
 		for f in self.funcs:
 			s += f.c_strspec(self.nam) + ',\n\n'
 
-		if self.init != None:
+		if self.event != None:
 			s += "\t/* Init/Fini */\n"
-			s += '\t"INIT\\0Vmod_' + self.nam + '_Func._init",\n'
+			s += '\t"$EVENT\\0Vmod_' + self.nam + '_Func._event",\n'
 
 		s += "\t0\n"
 		s += "};\n"
@@ -525,7 +526,7 @@ class Obj(object):
 
 	def c_strspec(self, modnam):
 		s = "\t/* Object " + self.nam + " */\n"
-		s += '\t"OBJ\\0"\n'
+		s += '\t"$OBJ\\0"\n'
 		s += self.init.c_strspec(modnam, pfx="\t\t") + '\n'
 		s += '\t\t"' + self.st + '\\0"\n'
 		s += self.fini.c_strspec(modnam, pfx="\t\t") + '\n'
@@ -749,9 +750,9 @@ class FileSection(object):
 		if t.str == "$Module":
 			o = parse_module(self)
 			vx.append(o)
-		elif t.str == "$Init":
+		elif t.str == "$Event":
 			x = self.get_token()
-			vx[0].set_init(x.str)
+			vx[0].set_event(x.str)
 			o = None
 		elif t.str == "$Function":
 			if len(vx) == 2:
@@ -854,7 +855,7 @@ def runmain(inputvcc, outputname="vcc_if"):
 		"$Function":	True,
 		"$Object":	True,
 		"$Method":	True,
-		"$Init":	True,
+		"$Event":	True,
 	}
 
 	sl = []
@@ -903,6 +904,7 @@ def runmain(inputvcc, outputname="vcc_if"):
 	vx[0].c_proto(fh)
 
 	fc.write('#include "config.h"\n')
+	fc.write('#include "vcl.h"\n')
 	fc.write('#include "vrt.h"\n')
 	fc.write('#include "vcc_if.h"\n')
 	fc.write('#include "vmod_abi.h"\n')
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 2e08154..67edd62 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -34,7 +34,7 @@ This vmod is used to develop, test and debug the various aspects
 of VMOD handling in Varnish.
 
 
-$Init init_function
+$Event event_function
 $Function VOID panic(STRING_LIST)
 
 Don't.
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index a91e845..f9f3361 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -248,12 +248,14 @@ priv_vcl_free(void *priv)
 }
 
 int __match_proto__(vmod_init_f)
-init_function(VRT_CTX, struct vmod_priv *priv)
+event_function(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
 {
 	struct priv_vcl *priv_vcl;
 
-	AN(ctx->msg);
+	if (e != VCL_EVENT_LOAD)
+		return (0);
 
+	AN(ctx->msg);
 	if (cache_param->nuke_limit == 42) {
 		VSB_printf(ctx->msg, "nuke_limit is not the answer.");
 		return (-1);



From phk at FreeBSD.org  Tue Jun 16 11:13:02 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jun 2015 13:13:02 +0200
Subject: [master] 5175539 Also call vmod::event on cold/warm/use events.
Message-ID: <E1Z4one-0006ul-Kk@project.varnish-software.com>


commit 5175539a4ddd8f266ad9d4d69b4bfc8529e2e5c6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 16 11:12:30 2015 +0000

    Also call vmod::event on cold/warm/use events.

diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index ffb70b7..e7a7f4f 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -205,6 +205,9 @@ vcc_ParseImport(struct vcc *tl)
 			VSB_printf(ifp->fin,
 			    "\t\t(void)%s(ctx, &vmod_priv_%.*s,\n"
 			    "\t\t    VCL_EVENT_DISCARD);\n", p, PF(mod));
+			VSB_printf(ifp->event,
+			    "\t(void)%s(ctx, &vmod_priv_%.*s, ev);\n",
+			    p, PF(mod));
 		} else {
 			sym = VCC_AddSymbolStr(tl, p, SYM_FUNC);
 			ERRCHK(tl);



From martin at varnish-software.com  Tue Jun 16 12:23:47 2015
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 16 Jun 2015 14:23:47 +0200
Subject: [master] 73c6012 Clearify the transaction grouping documentation
Message-ID: <E1Z4pu7-0004sa-Tj@project.varnish-software.com>


commit 73c601204a3faa8d8faeb0184f51d17f08afbe58
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 16 14:23:22 2015 +0200

    Clearify the transaction grouping documentation

diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst
index 11b40bd..3b31c49 100644
--- a/doc/sphinx/reference/vsl-query.rst
+++ b/doc/sphinx/reference/vsl-query.rst
@@ -50,15 +50,18 @@ The grouping modes are:
 * Session
 
   All transactions initiated by a client connection are reported
-  together. All log data is buffered until the client connection is
-  closed, which can cause session grouping mode to potentially consume
-  a lot of memory.
+  together. Client connections are open ended when using HTTP
+  keep-alives, so it is undefined when the session will be
+  reported. If the transaction timeout period is exceeded an
+  incomplete session will be reported. Non-transactional data (VXID
+  == 0) is not reported.
 
 * Request
 
   Transactions are grouped by request, where the set will include the
   request itself as well as any backend requests or ESI-subrequests.
-  Session data is not reported. This is the default.
+  Session data and non-transactional data (VXID == 0) is not
+  reported. This is the default.
 
 * VXID
 
@@ -72,7 +75,16 @@ The grouping modes are:
   Every log record will make up a transaction of it's own. All data,
   including non-transactional data will be reported.
 
-Example transaction hierarchy ::
+The API will when possible use shared memory pointers for the log
+data. This keeps the memory usage low as there will be no local
+buffering of log data. Though the shared memory log is a ring buffer,
+and the data will be overwritten on the next wrap of the ring
+buffer. The API will start to buffer data when the Varnish daemon
+comes close to overwriting still unreported content. When using
+session grouping together with long lasting client connections the
+memory usage of the logging process can increase.
+
+Example transaction hierarchy using request grouping mode ::
 
   Lvl 1: Client request (cache miss)
     Lvl 2: Backend request



From nils.goroll at uplex.de  Tue Jun 16 12:57:41 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 16 Jun 2015 14:57:41 +0200
Subject: [master] 3f1a5ea reword the vsl memory usage section a bit
Message-ID: <E1Z4qQv-0002Np-C2@project.varnish-software.com>


commit 3f1a5eae5a23f0d56331eebd8ba0ab7f3d9d6157
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 16 14:57:32 2015 +0200

    reword the vsl memory usage section a bit

diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst
index 3b31c49..6245a06 100644
--- a/doc/sphinx/reference/vsl-query.rst
+++ b/doc/sphinx/reference/vsl-query.rst
@@ -35,12 +35,11 @@ GROUPING
 ========
 
 When grouping transactions, there is a hierarchy structure showing
-which transaction initiated what. The level increases by one by an
+which transaction initiated what. The level increases by one on an
 'initiated by' relation, so for example a backend transaction will
 have one higher level than the client transaction that initiated it on
-a cache miss. Request restart transactions does not have it's level
-increased. This is to help predicting the level for a given
-transaction.
+a cache miss. Request restart transactions don't get their level
+increased to make it predictable.
 
 Levels start counting at 1, except when using raw where it will always
 be 0.
@@ -75,14 +74,27 @@ The grouping modes are:
   Every log record will make up a transaction of it's own. All data,
   including non-transactional data will be reported.
 
-The API will when possible use shared memory pointers for the log
-data. This keeps the memory usage low as there will be no local
-buffering of log data. Though the shared memory log is a ring buffer,
-and the data will be overwritten on the next wrap of the ring
-buffer. The API will start to buffer data when the Varnish daemon
-comes close to overwriting still unreported content. When using
-session grouping together with long lasting client connections the
-memory usage of the logging process can increase.
+Memory Usage
+------------
+
+The API will use pointers to shared memory log data as long as
+possible to keep memory usage at a minimum. But as the shared memory
+log is a ring buffer, data will get overwritten eventually, so the API
+creates local copies of referenced log data when varnishd comes close
+to overwriting still unreported content.
+
+This process avoids loss of log data in many scenarios, but it is not
+failsafe: Overruns where varnishd "overtakes" the log reader process
+in the ring buffer can still happen when API clients cannot keep up
+reading and/or copying, for instance due to output blocking.
+
+Though being unrelated to grouping in principle, copying of log data
+is particularly relevant for session grouping together with long
+lasting client connections - for this grouping, the logging API client
+process is likely to consume relevant amounts of memory. As the vxid
+grouping also logs (potentially long lasting) sessions, it is also
+likely to require memory for copies of log entries, but far less than
+session grouping.
 
 Example transaction hierarchy using request grouping mode ::
 



From nils.goroll at uplex.de  Tue Jun 16 13:09:55 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 16 Jun 2015 15:09:55 +0200
Subject: [master] 3bd031e reorder sections
Message-ID: <E1Z4qcl-00061G-KH@project.varnish-software.com>


commit 3bd031e336c8db3402a4b948a034ad60a4f7e651
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 16 15:09:39 2015 +0200

    reorder sections

diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst
index 6245a06..864af0a 100644
--- a/doc/sphinx/reference/vsl-query.rst
+++ b/doc/sphinx/reference/vsl-query.rst
@@ -74,6 +74,20 @@ The grouping modes are:
   Every log record will make up a transaction of it's own. All data,
   including non-transactional data will be reported.
 
+Transaction Hierarchy
+---------------------
+
+Example transaction hierarchy using request grouping mode ::
+
+  Lvl 1: Client request (cache miss)
+    Lvl 2: Backend request
+    Lvl 2: ESI subrequest (cache miss)
+      Lvl 3: Backend request
+      Lvl 3: Backend request (VCL restart)
+      Lvl 3: ESI subrequest (cache miss)
+        Lvl 4: Backend request
+    Lvl 2: ESI subrequest (cache hit)
+
 Memory Usage
 ------------
 
@@ -96,17 +110,6 @@ grouping also logs (potentially long lasting) sessions, it is also
 likely to require memory for copies of log entries, but far less than
 session grouping.
 
-Example transaction hierarchy using request grouping mode ::
-
-  Lvl 1: Client request (cache miss)
-    Lvl 2: Backend request
-    Lvl 2: ESI subrequest (cache miss)
-      Lvl 3: Backend request
-      Lvl 3: Backend request (VCL restart)
-      Lvl 3: ESI subrequest (cache miss)
-        Lvl 4: Backend request
-    Lvl 2: ESI subrequest (cache hit)
-
 QUERY LANGUAGE
 ==============
 



From nils.goroll at uplex.de  Tue Jun 16 13:15:39 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 16 Jun 2015 15:15:39 +0200
Subject: [master] 8336ff8 up one level (unrelated to grouping)
Message-ID: <E1Z4qiJ-0000e9-Th@project.varnish-software.com>


commit 8336ff8af5786bcbe4948894af305c6a7c4c657e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 16 15:15:35 2015 +0200

    up one level (unrelated to grouping)

diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst
index 864af0a..ec73724 100644
--- a/doc/sphinx/reference/vsl-query.rst
+++ b/doc/sphinx/reference/vsl-query.rst
@@ -88,8 +88,8 @@ Example transaction hierarchy using request grouping mode ::
         Lvl 4: Backend request
     Lvl 2: ESI subrequest (cache hit)
 
-Memory Usage
-------------
+MEMORY USAGE
+============
 
 The API will use pointers to shared memory log data as long as
 possible to keep memory usage at a minimum. But as the shared memory



From phk at FreeBSD.org  Wed Jun 17 07:55:00 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 09:55:00 +0200
Subject: [master] 95c2094 Polish the generated C code a bit, new order:
Message-ID: <E1Z58BY-0007nk-Uk@project.varnish-software.com>


commit 95c209470655ca6a8339c854368e1c3b5d8d5b4b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 07:53:18 2015 +0000

    Polish the generated C code a bit, new order:
    
    	Boilerplate .h files.
    	Source code copy
    	Location tables
    	Generated .h
    	Generated .c
    
    Added a few more "---===###" separators to make this clear

diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 21fc395..34657d4 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -250,15 +250,41 @@ EncToken(struct vsb *sb, const struct token *t)
  */
 
 static void
-LocTable(const struct vcc *tl)
+EmitCoordinates(const struct vcc *tl, struct vsb *vsb)
 {
 	struct token *t;
 	unsigned lin, pos;
 	struct source *sp;
 	const char *p;
 
-	Fh(tl, 0, "\n#define VGC_NREFS %u\n", tl->cnt + 1);
-	Fc(tl, 0, "\nstatic struct vrt_ref VGC_ref[VGC_NREFS] = {\n");
+	VSB_printf(vsb, "/* ---===### Source Code ###===---*/\n");
+
+	VSB_printf(vsb, "\n#define VGC_NSRCS %u\n", tl->nsources);
+
+	VSB_printf(vsb, "\nstatic const char *srcname[VGC_NSRCS] = {\n");
+	VTAILQ_FOREACH(sp, &tl->sources, list) {
+		VSB_printf(vsb, "\t");
+		EncString(vsb, sp->name, NULL, 0);
+		VSB_printf(vsb, ",\n");
+	}
+	VSB_printf(vsb, "};\n");
+
+	VSB_printf(vsb, "\nstatic const char *srcbody[%u] = {\n", tl->nsources);
+	VTAILQ_FOREACH(sp, &tl->sources, list) {
+		VSB_printf(vsb, "    /* ");
+		EncString(vsb, sp->name, NULL, 0);
+		VSB_printf(vsb, "*/\n");
+		VSB_printf(vsb, "\t");
+		EncString(vsb, sp->b, sp->e, 1);
+		VSB_printf(vsb, ",\n");
+	}
+	VSB_printf(vsb, "};\n\n");
+
+	VSB_printf(vsb, "/* ---===### Location Counters ###===---*/\n");
+
+	VSB_printf(vsb, "\n#define VGC_NREFS %u\n", tl->cnt + 1);
+
+	VSB_printf(vsb, "\nstatic struct vrt_ref VGC_ref[VGC_NREFS] = {\n");
 	lin = 1;
 	pos = 0;
 	sp = 0;
@@ -286,14 +312,14 @@ LocTable(const struct vcc *tl)
 				pos++;
 
 		}
-		Fc(tl, 0, "  [%3u] = { %d, %8tu, %4u, %3u, 0, ",
+		VSB_printf(vsb, "  [%3u] = { %d, %8tu, %4u, %3u, 0, ",
 		    t->cnt, sp->idx, t->b - sp->b, lin, pos + 1);
 		if (t->tok == CSRC)
-			Fc(tl, 0, " \"C{\"},\n");
+			VSB_printf(vsb, " \"C{\"},\n");
 		else
-			Fc(tl, 0, " \"%.*s\" },\n", PF(t));
+			VSB_printf(vsb, " \"%.*s\" },\n", PF(t));
 	}
-	Fc(tl, 0, "};\n");
+	VSB_printf(vsb, "};\n\n");
 }
 
 /*--------------------------------------------------------------------
@@ -375,36 +401,13 @@ EmitInitFini(const struct vcc *tl)
 static void
 EmitStruct(const struct vcc *tl)
 {
-	struct source *sp;
-
-	Fc(tl, 0, "\nextern const char *srcname[];\n");
-	Fc(tl, 0, "\nconst char *srcname[%u] = {\n", tl->nsources);
-	VTAILQ_FOREACH(sp, &tl->sources, list) {
-		Fc(tl, 0, "\t");
-		EncString(tl->fc, sp->name, NULL, 0);
-		Fc(tl, 0, ",\n");
-	}
-	Fc(tl, 0, "};\n");
-
-	Fc(tl, 0, "\nextern const char *srcbody[];\n");
-	Fc(tl, 0, "\nconst char *srcbody[%u] = {\n", tl->nsources);
-	VTAILQ_FOREACH(sp, &tl->sources, list) {
-		Fc(tl, 0, "    /* ");
-		EncString(tl->fc, sp->name, NULL, 0);
-		Fc(tl, 0, "*/\n");
-		Fc(tl, 0, "\t");
-		EncString(tl->fc, sp->b, sp->e, 1);
-		Fc(tl, 0, ",\n");
-	}
-	Fc(tl, 0, "};\n");
-
 	Fc(tl, 0, "\nconst struct VCL_conf VCL_conf = {\n");
 	Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n");
 	Fc(tl, 0, "\t.event_vcl = VGC_Event,\n");
 	Fc(tl, 0, "\t.default_director = &%s,\n", tl->default_director);
 	Fc(tl, 0, "\t.ref = VGC_ref,\n");
 	Fc(tl, 0, "\t.nref = VGC_NREFS,\n");
-	Fc(tl, 0, "\t.nsrc = %u,\n", tl->nsources);
+	Fc(tl, 0, "\t.nsrc = VGC_NSRCS,\n");
 	Fc(tl, 0, "\t.srcname = srcname,\n");
 	Fc(tl, 0, "\t.srcbody = srcbody,\n");
 #define VCL_MET_MAC(l,u,b) \
@@ -606,12 +609,17 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	struct vcc *tl;
 	struct symbol *sym;
 	const struct var *v;
+	struct vsb *vsb;
+
 	char *of;
 	int i;
 
 	tl = vcc_NewVcc(tl0);
 	tl->sb = sb;
 
+	vsb = VSB_new_auto();
+	AN(vsb);
+
 	vcc_Expr_Init(tl);
 
 	for (v = tl->vars; v->name != NULL; v++) {
@@ -630,8 +638,9 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	sym = VCC_AddSymbolStr(tl, "storage.", SYM_WILDCARD);
 	sym->wildcard = vcc_Stv_Wildcard;
 
-	vcl_output_lang_h(tl->fh);
-	Fh(tl, 0, "/* ---===### VCC generated code ###===---*/\n");
+	vcl_output_lang_h(vsb);
+	Fh(tl, 0, "/* ---===### VCC generated .h code ###===---*/\n");
+	Fc(tl, 0, "\n/* ---===### VCC generated .c code ###===---*/\n");
 	Fh(tl, 0, "\nextern const struct VCL_conf VCL_conf;\n");
 
 	/* Register and lex the main source */
@@ -691,9 +700,10 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 		return (vcc_DestroyTokenList(tl, NULL));
 
 	/* Emit method functions */
+	Fh(tl, 1, "\n");
 	for (i = 1; i < VCL_MET_MAX; i++) {
-		Fh(tl, 1, "\nint __match_proto__(vcl_func_f)\n");
 		Fh(tl, 1,
+		    "int __match_proto__(vcl_func_f) "
 		    "VGC_function_%s(VRT_CTX);\n",
 		    method_tab[i].name);
 		Fc(tl, 1, "\nint __match_proto__(vcl_func_f)\n");
@@ -715,20 +725,27 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 		Fc(tl, 1, "}\n");
 	}
 
-	LocTable(tl);
-
 	EmitInitFini(tl);
 
 	EmitStruct(tl);
 
-	/* Combine it all in the fh vsb */
-	AZ(VSB_finish(tl->fc));
-	VSB_cat(tl->fh, VSB_data(tl->fc));
+	/* Combine it all in the vsb */
+
+	EmitCoordinates(tl, vsb);
+
 	AZ(VSB_finish(tl->fh));
+	VSB_cat(vsb, VSB_data(tl->fh));
 
-	of = strdup(VSB_data(tl->fh));
+	AZ(VSB_finish(tl->fc));
+	VSB_cat(vsb, VSB_data(tl->fc));
+
+	AZ(VSB_finish(vsb));
+
+	of = strdup(VSB_data(vsb));
 	AN(of);
 
+	VSB_delete(vsb);
+
 	/* done */
 	return (vcc_DestroyTokenList(tl, of));
 }



From phk at FreeBSD.org  Wed Jun 17 07:55:00 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 09:55:00 +0200
Subject: [master] aa19241 Merge branch 'master' of
 git.varnish-cache.org:varnish-cache
Message-ID: <E1Z58BY-0007nr-Vu@project.varnish-software.com>


commit aa192411a85e0c2cde0cf8954d5668d8639c118f
Merge: 95c2094 8336ff8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 07:54:55 2015 +0000

    Merge branch 'master' of git.varnish-cache.org:varnish-cache



From phk at FreeBSD.org  Wed Jun 17 09:39:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 11:39:04 +0200
Subject: [master] 3018d44 FlexeLint nit-pickery
Message-ID: <E1Z59oG-0000uB-23@project.varnish-software.com>


commit 3018d44ab4e413efcbda8aae4a5d60b0fd238fdf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 08:10:38 2015 +0000

    FlexeLint nit-pickery

diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 34657d4..f21045b 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -312,7 +312,7 @@ EmitCoordinates(const struct vcc *tl, struct vsb *vsb)
 				pos++;
 
 		}
-		VSB_printf(vsb, "  [%3u] = { %d, %8tu, %4u, %3u, 0, ",
+		VSB_printf(vsb, "  [%3u] = { %u, %8tu, %4u, %3u, 0, ",
 		    t->cnt, sp->idx, t->b - sp->b, lin, pos + 1);
 		if (t->tok == CSRC)
 			VSB_printf(vsb, " \"C{\"},\n");
@@ -617,9 +617,6 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	tl = vcc_NewVcc(tl0);
 	tl->sb = sb;
 
-	vsb = VSB_new_auto();
-	AN(vsb);
-
 	vcc_Expr_Init(tl);
 
 	for (v = tl->vars; v->name != NULL; v++) {
@@ -638,7 +635,6 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	sym = VCC_AddSymbolStr(tl, "storage.", SYM_WILDCARD);
 	sym->wildcard = vcc_Stv_Wildcard;
 
-	vcl_output_lang_h(vsb);
 	Fh(tl, 0, "/* ---===### VCC generated .h code ###===---*/\n");
 	Fc(tl, 0, "\n/* ---===### VCC generated .c code ###===---*/\n");
 	Fh(tl, 0, "\nextern const struct VCL_conf VCL_conf;\n");
@@ -729,7 +725,12 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 
 	EmitStruct(tl);
 
-	/* Combine it all in the vsb */
+	/* Combine it all */
+
+	vsb = VSB_new_auto();
+	AN(vsb);
+
+	vcl_output_lang_h(vsb);
 
 	EmitCoordinates(tl, vsb);
 



From phk at FreeBSD.org  Wed Jun 17 09:39:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 11:39:04 +0200
Subject: [master] 2006bd9 Rename struct vcls to struct vcl
Message-ID: <E1Z59oG-0000uH-3J@project.varnish-software.com>


commit 2006bd9d73234388ec201ac211fd641fff07bbd3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 08:10:59 2015 +0000

    Rename struct vcls to struct vcl

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 046b731..bea6a90 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -44,10 +44,10 @@
 #include "vcli.h"
 #include "vcli_priv.h"
 
-struct vcls {
+struct vcl {
 	unsigned		magic;
 #define VVCLS_MAGIC		0x214188f2
-	VTAILQ_ENTRY(vcls)	list;
+	VTAILQ_ENTRY(vcl)	list;
 	void			*dlh;
 	struct VCL_conf		conf[1];
 	char			state[8];
@@ -58,11 +58,11 @@ struct vcls {
  * XXX: Presently all modifications to this list happen from the
  * CLI event-engine, so no locking is necessary
  */
-static VTAILQ_HEAD(, vcls)	vcl_head =
+static VTAILQ_HEAD(, vcl)	vcl_head =
     VTAILQ_HEAD_INITIALIZER(vcl_head);
 
 static struct lock		vcl_mtx;
-static struct vcls		*vcl_active; /* protected by vcl_mtx */
+static struct vcl		*vcl_active; /* protected by vcl_mtx */
 
 /*--------------------------------------------------------------------*/
 
@@ -154,10 +154,10 @@ VCL_Rel(struct VCL_conf **vcc)
 
 /*--------------------------------------------------------------------*/
 
-static struct vcls *
+static struct vcl *
 VCL_Open(const char *fn, struct vsb *msg)
 {
-	struct vcls *vcl;
+	struct vcl *vcl;
 	void *dlh;
 	struct VCL_conf const *cnf;
 
@@ -189,9 +189,9 @@ VCL_Open(const char *fn, struct vsb *msg)
 }
 
 static void
-VCL_Close(struct vcls **vclp)
+VCL_Close(struct vcl **vclp)
 {
-	struct vcls *vcl;
+	struct vcl *vcl;
 
 	CHECK_OBJ_NOTNULL(*vclp, VVCLS_MAGIC);
 	vcl = *vclp;
@@ -208,7 +208,7 @@ int
 VCL_TestLoad(const char *fn)
 {
 	struct vsb *vsb;
-	struct vcls *vcl;
+	struct vcl *vcl;
 	int retval = 0;
 
 	AN(fn);
@@ -227,10 +227,10 @@ VCL_TestLoad(const char *fn)
 
 /*--------------------------------------------------------------------*/
 
-static struct vcls *
+static struct vcl *
 vcl_find(const char *name)
 {
-	struct vcls *vcl;
+	struct vcl *vcl;
 
 	ASSERT_CLI();
 	VTAILQ_FOREACH(vcl, &vcl_head, list) {
@@ -243,7 +243,7 @@ vcl_find(const char *name)
 }
 
 static void
-vcl_set_state(struct vcls *vcl, const char *state)
+vcl_set_state(struct vcl *vcl, const char *state)
 {
 	struct vrt_ctx ctx;
 	int warm;
@@ -266,7 +266,7 @@ vcl_set_state(struct vcls *vcl, const char *state)
 static int
 VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 {
-	struct vcls *vcl;
+	struct vcl *vcl;
 	struct vrt_ctx ctx;
 	unsigned hand = 0;
 	struct vsb *vsb;
@@ -332,7 +332,7 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
  */
 
 static void
-VCL_Nuke(struct vcls *vcl)
+VCL_Nuke(struct vcl *vcl)
 {
 	struct vrt_ctx ctx;
 	unsigned hand = 0;
@@ -358,7 +358,7 @@ VCL_Nuke(struct vcls *vcl)
 void
 VCL_Poll(void)
 {
-	struct vcls *vcl, *vcl2;
+	struct vcl *vcl, *vcl2;
 
 	ASSERT_CLI();
 	VTAILQ_FOREACH_SAFE(vcl, &vcl_head, list, vcl2)
@@ -371,7 +371,7 @@ VCL_Poll(void)
 static void __match_proto__(cli_func_t)
 ccf_config_list(struct cli *cli, const char * const *av, void *priv)
 {
-	struct vcls *vcl;
+	struct vcl *vcl;
 	const char *flg;
 
 	(void)av;
@@ -406,7 +406,7 @@ ccf_config_load(struct cli *cli, const char * const *av, void *priv)
 static void __match_proto__(cli_func_t)
 ccf_config_state(struct cli *cli, const char * const *av, void *priv)
 {
-	struct vcls *vcl;
+	struct vcl *vcl;
 
 	(void)cli;
 	AZ(priv);
@@ -421,7 +421,7 @@ ccf_config_state(struct cli *cli, const char * const *av, void *priv)
 static void __match_proto__(cli_func_t)
 ccf_config_discard(struct cli *cli, const char * const *av, void *priv)
 {
-	struct vcls *vcl;
+	struct vcl *vcl;
 
 	ASSERT_CLI();
 	(void)cli;
@@ -442,7 +442,7 @@ ccf_config_discard(struct cli *cli, const char * const *av, void *priv)
 static void __match_proto__(cli_func_t)
 ccf_config_use(struct cli *cli, const char * const *av, void *priv)
 {
-	struct vcls *vcl;
+	struct vcl *vcl;
 	struct vrt_ctx ctx;
 	unsigned hand = 0;
 	struct vsb *vsb;
@@ -477,7 +477,7 @@ ccf_config_use(struct cli *cli, const char * const *av, void *priv)
 static void __match_proto__(cli_func_t)
 ccf_config_show(struct cli *cli, const char * const *av, void *priv)
 {
-	struct vcls *vcl;
+	struct vcl *vcl;
 	int verbose = 0;
 	int i;
 



From phk at FreeBSD.org  Wed Jun 17 09:39:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 11:39:04 +0200
Subject: [master] 2c128ca Move the panic-dump of VCL's to cache_vcl.c so
 cache_panic.c doesn't need to know what's inside.
Message-ID: <E1Z59oG-0000uR-4H@project.varnish-software.com>


commit 2c128ca1917f434e32008c350a158ab49517f958
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 08:34:29 2015 +0000

    Move the panic-dump of VCL's to cache_vcl.c so cache_panic.c doesn't
    need to know what's inside.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a29d998..3492ae5 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1066,6 +1066,7 @@ void VRY_Finish(struct req *req, enum vry_finish_flag);
 
 /* cache_vcl.c */
 void VCL_Init(void);
+void VCL_Panic(struct vsb *, const struct VCL_conf *);
 void VCL_Refresh(struct VCL_conf **vcc);
 void VCL_Ref(struct VCL_conf *vcc);
 void VCL_Rel(struct VCL_conf **vcc);
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 6f59a5e..3755508 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -45,7 +45,6 @@
 
 #include "cache_backend.h"
 #include "storage/storage.h"
-#include "vcl.h"
 #include "vcli_priv.h"
 
 /*
@@ -257,22 +256,6 @@ pan_objcore(const char *typ, const struct objcore *oc)
 /*--------------------------------------------------------------------*/
 
 static void
-pan_vcl(const struct VCL_conf *vcl)
-{
-	int i;
-
-	VSB_printf(pan_vsp, "  vcl = {\n");
-	VSB_printf(pan_vsp, "    srcname = {\n");
-	for (i = 0; i < vcl->nsrc; ++i)
-		VSB_printf(pan_vsp, "      \"%s\",\n", vcl->srcname[i]);
-	VSB_printf(pan_vsp, "    },\n");
-	VSB_printf(pan_vsp, "  },\n");
-}
-
-
-/*--------------------------------------------------------------------*/
-
-static void
 pan_wrk(const struct worker *wrk)
 {
 	const char *hand;
@@ -406,8 +389,7 @@ pan_req(const struct req *req)
 	if (req->resp->ws != NULL)
 		pan_http("resp", req->resp, 2);
 
-	if (VALID_OBJ(req->vcl, VCL_CONF_MAGIC))
-		pan_vcl(req->vcl);
+	VCL_Panic(pan_vsp, req->vcl);
 
 	if (req->objcore != NULL) {
 		pan_objcore("REQ", req->objcore);
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index bea6a90..db5aa2d 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -66,6 +66,24 @@ static struct vcl		*vcl_active; /* protected by vcl_mtx */
 
 /*--------------------------------------------------------------------*/
 
+void
+VCL_Panic(struct vsb *vsb, const struct VCL_conf *vcl)
+{
+	int i;
+
+	AN(vsb);
+	if (vcl == NULL)
+		return;
+	VSB_printf(vsb, "  vcl = {\n");
+	VSB_printf(vsb, "    srcname = {\n");
+	for (i = 0; i < vcl->nsrc; ++i)
+		VSB_printf(vsb, "      \"%s\",\n", vcl->srcname[i]);
+	VSB_printf(vsb, "    },\n");
+	VSB_printf(vsb, "  },\n");
+}
+
+/*--------------------------------------------------------------------*/
+
 const char *
 VCL_Return_Name(unsigned r)
 {



From phk at FreeBSD.org  Wed Jun 17 09:39:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 11:39:04 +0200
Subject: [master] 93112f3 Isolate more knowledge about the internals of a VCL
 in cache_vcl.c
Message-ID: <E1Z59oG-0000ua-5N@project.varnish-software.com>


commit 93112f34d060afbb290541c699c2cd86214d8708
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 08:44:18 2015 +0000

    Isolate more knowledge about the internals of a VCL in cache_vcl.c

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 3492ae5..042e176 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1071,6 +1071,7 @@ void VCL_Refresh(struct VCL_conf **vcc);
 void VCL_Ref(struct VCL_conf *vcc);
 void VCL_Rel(struct VCL_conf **vcc);
 void VCL_Poll(void);
+struct director *VCL_DefaultDirector(const struct VCL_conf *);
 const char *VCL_Return_Name(unsigned);
 const char *VCL_Method_Name(unsigned);
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c4d3ab4..5ce8309 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -107,7 +107,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore->objhead, OBJHEAD_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AN(req->vcl);
 
 	assert(req->objcore->refcnt > 0);
 
@@ -337,7 +337,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	AZ(req->objcore);
 
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AN(req->vcl);
 
 	VRY_Prep(req);
 
@@ -469,7 +469,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AN(req->vcl);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
 	VCL_miss_method(req->vcl, wrk, req, NULL, NULL);
@@ -510,7 +510,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AN(req->vcl);
 	AZ(req->objcore);
 
 	VCL_pass_method(req->vcl, wrk, req, NULL, NULL);
@@ -545,7 +545,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AN(req->vcl);
 
 	wrk->stats->s_pipe++;
 	bo = VBO_GetBusyObj(wrk, req);
@@ -618,7 +618,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AN(req->vcl);
 	AZ(req->objcore);
 	AZ(req->err_code);
 
@@ -649,7 +649,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 
 	/* By default we use the first backend */
 	AZ(req->director_hint);
-	req->director_hint = *req->vcl->default_director;
+	req->director_hint = VCL_DefaultDirector(req->vcl);
 	AN(req->director_hint);
 
 	req->d_ttl = -1;
@@ -735,7 +735,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	AZ(req->objcore);
 
-	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AN(req->vcl);
 
 	VRY_Prep(req);
 
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index db5aa2d..e45f186 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -245,6 +245,16 @@ VCL_TestLoad(const char *fn)
 
 /*--------------------------------------------------------------------*/
 
+struct director *
+VCL_DefaultDirector(const struct VCL_conf *vcc)
+{
+
+	AN(vcc);
+	return (*vcc->default_director);
+}
+
+/*--------------------------------------------------------------------*/
+
 static struct vcl *
 vcl_find(const char *name)
 {



From phk at FreeBSD.org  Wed Jun 17 09:39:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 11:39:04 +0200
Subject: [master] fa6d649 Isolate VCL-internals more in cache_vcl.c
Message-ID: <E1Z59oG-0000uj-6Q@project.varnish-software.com>


commit fa6d649095d8b3018a92d48bad7c6ffe83e10b2b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 08:51:47 2015 +0000

    Isolate VCL-internals more in cache_vcl.c

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 042e176..e8836fb 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1065,15 +1065,16 @@ enum vry_finish_flag { KEEP, DISCARD };
 void VRY_Finish(struct req *req, enum vry_finish_flag);
 
 /* cache_vcl.c */
+struct director *VCL_DefaultDirector(const struct VCL_conf *);
 void VCL_Init(void);
+const char *VCL_Method_Name(unsigned);
+const char *VCL_Name(const struct VCL_conf *);
 void VCL_Panic(struct vsb *, const struct VCL_conf *);
-void VCL_Refresh(struct VCL_conf **vcc);
-void VCL_Ref(struct VCL_conf *vcc);
-void VCL_Rel(struct VCL_conf **vcc);
 void VCL_Poll(void);
-struct director *VCL_DefaultDirector(const struct VCL_conf *);
+void VCL_Ref(struct VCL_conf *);
+void VCL_Refresh(struct VCL_conf **);
+void VCL_Rel(struct VCL_conf **);
 const char *VCL_Return_Name(unsigned);
-const char *VCL_Method_Name(unsigned);
 
 #define VCL_MET_MAC(l,u,b) \
     void VCL_##l##_method(struct VCL_conf *, struct worker *, struct req *, \
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 9fa64b5..fc15baa 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -41,7 +41,6 @@
 #include "cache_backend.h"
 #include "vcli.h"
 #include "vcli_priv.h"
-#include "vcl.h"
 #include "vsa.h"
 #include "vrt.h"
 #include "vtim.h"
@@ -162,13 +161,13 @@ backend_find(struct cli *cli, const char *matcher, bf_func *func, void *priv)
 	AN(vsb);
 	if (matcher == NULL || *matcher == '\0' || !strcmp(matcher, "*")) {
 		// all backends in active VCL
-		VSB_printf(vsb, "%s.*", vcc->loaded_name);
+		VSB_printf(vsb, "%s.*", VCL_Name(vcc));
 	} else if (strchr(matcher, '.') != NULL) {
 		// use pattern as is
 		VSB_cat(vsb, matcher);
 	} else {
 		// pattern applies to active vcl
-		VSB_printf(vsb, "%s.%s", vcc->loaded_name, matcher);
+		VSB_printf(vsb, "%s.%s", VCL_Name(vcc), matcher);
 	}
 	AZ(VSB_finish(vsb));
 	Lck_Lock(&backends_mtx);
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index e45f186..bca73c2 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -253,6 +253,13 @@ VCL_DefaultDirector(const struct VCL_conf *vcc)
 	return (*vcc->default_director);
 }
 
+const char *
+VCL_Name(const struct VCL_conf *vcc)
+{
+	AN(vcc);
+	return (vcc->loaded_name);
+}
+
 /*--------------------------------------------------------------------*/
 
 static struct vcl *



From phk at FreeBSD.org  Wed Jun 17 09:39:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 11:39:04 +0200
Subject: [master] 08fe2b8 Make the reference req/busyobj and a few other bits
 hold in VCL's a pointer to our internal housekeeping structure,
 rather than the VCL_conf stucture.
Message-ID: <E1Z59oG-0000uu-7W@project.varnish-software.com>


commit 08fe2b87a62a68fa3629619aac693f2b8467c2ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 08:57:52 2015 +0000

    Make the reference req/busyobj and a few other bits hold in VCL's
    a pointer to our internal housekeeping structure, rather than the
    VCL_conf stucture.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index e8836fb..49bf6ba 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -341,7 +341,7 @@ struct worker {
 
 	pthread_cond_t		cond;
 
-	struct VCL_conf		*vcl;
+	struct vcl		*vcl;
 
 	struct ws		aws[1];
 
@@ -503,7 +503,7 @@ struct busyobj {
 	const struct director	*director_req;
 	const struct director	*director_resp;
 	enum director_state_e	director_state;
-	struct VCL_conf		*vcl;
+	struct vcl		*vcl;
 
 	struct vsl_log		vsl[1];
 
@@ -560,7 +560,7 @@ struct req {
 	const char		*err_reason;
 
 	const struct director	*director_hint;
-	struct VCL_conf		*vcl;
+	struct vcl		*vcl;
 
 	char			*ws_req;	/* WS above request data */
 
@@ -1065,19 +1065,19 @@ enum vry_finish_flag { KEEP, DISCARD };
 void VRY_Finish(struct req *req, enum vry_finish_flag);
 
 /* cache_vcl.c */
-struct director *VCL_DefaultDirector(const struct VCL_conf *);
+struct director *VCL_DefaultDirector(const struct vcl *);
 void VCL_Init(void);
 const char *VCL_Method_Name(unsigned);
-const char *VCL_Name(const struct VCL_conf *);
-void VCL_Panic(struct vsb *, const struct VCL_conf *);
+const char *VCL_Name(const struct vcl *);
+void VCL_Panic(struct vsb *, const struct vcl *);
 void VCL_Poll(void);
-void VCL_Ref(struct VCL_conf *);
-void VCL_Refresh(struct VCL_conf **);
-void VCL_Rel(struct VCL_conf **);
+void VCL_Ref(struct vcl *);
+void VCL_Refresh(struct vcl **);
+void VCL_Rel(struct vcl **);
 const char *VCL_Return_Name(unsigned);
 
 #define VCL_MET_MAC(l,u,b) \
-    void VCL_##l##_method(struct VCL_conf *, struct worker *, struct req *, \
+    void VCL_##l##_method(struct vcl *, struct worker *, struct req *, \
 	struct busyobj *bo, void *specific);
 #include "tbl/vcl_returns.h"
 #undef VCL_MET_MAC
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index fc15baa..9f831b4 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -152,7 +152,7 @@ backend_find(struct cli *cli, const char *matcher, bf_func *func, void *priv)
 {
 	int i, found = 0;
 	struct vsb *vsb;
-	struct VCL_conf *vcc = NULL;
+	struct vcl *vcc = NULL;
 	struct backend *b;
 
 	VCL_Refresh(&vcc);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3e13dd2..3eed33b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -978,7 +978,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc,
 
 	oc->busyobj = bo;
 
-	CHECK_OBJ_NOTNULL(bo->vcl, VCL_CONF_MAGIC);
+	AN(bo->vcl);
 
 	if (mode == VBF_PASS)
 		bo->do_pass = 1;
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index bca73c2..4f1ad1d 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -46,7 +46,7 @@
 
 struct vcl {
 	unsigned		magic;
-#define VVCLS_MAGIC		0x214188f2
+#define VCL_MAGIC		0x214188f2
 	VTAILQ_ENTRY(vcl)	list;
 	void			*dlh;
 	struct VCL_conf		conf[1];
@@ -67,7 +67,7 @@ static struct vcl		*vcl_active; /* protected by vcl_mtx */
 /*--------------------------------------------------------------------*/
 
 void
-VCL_Panic(struct vsb *vsb, const struct VCL_conf *vcl)
+VCL_Panic(struct vsb *vsb, const struct vcl *vcl)
 {
 	int i;
 
@@ -76,8 +76,8 @@ VCL_Panic(struct vsb *vsb, const struct VCL_conf *vcl)
 		return;
 	VSB_printf(vsb, "  vcl = {\n");
 	VSB_printf(vsb, "    srcname = {\n");
-	for (i = 0; i < vcl->nsrc; ++i)
-		VSB_printf(vsb, "      \"%s\",\n", vcl->srcname[i]);
+	for (i = 0; i < vcl->conf->nsrc; ++i)
+		VSB_printf(vsb, "      \"%s\",\n", vcl->conf->srcname[i]);
 	VSB_printf(vsb, "    },\n");
 	VSB_printf(vsb, "  },\n");
 }
@@ -113,7 +113,7 @@ VCL_Method_Name(unsigned m)
 /*--------------------------------------------------------------------*/
 
 static void
-VCL_Get(struct VCL_conf **vcc)
+VCL_Get(struct vcl **vcc)
 {
 	static int once = 0;
 
@@ -124,17 +124,17 @@ VCL_Get(struct VCL_conf **vcc)
 
 	Lck_Lock(&vcl_mtx);
 	AN(vcl_active);
-	*vcc = vcl_active->conf;
+	*vcc = vcl_active;
 	AN(*vcc);
-	AZ((*vcc)->discard);
-	(*vcc)->busy++;
+	AZ((*vcc)->conf->discard);
+	(*vcc)->conf->busy++;
 	Lck_Unlock(&vcl_mtx);
 }
 
 void
-VCL_Refresh(struct VCL_conf **vcc)
+VCL_Refresh(struct vcl **vcc)
 {
-	if (*vcc == vcl_active->conf)
+	if (*vcc == vcl_active)
 		return;
 	if (*vcc != NULL)
 		VCL_Rel(vcc);	/* XXX: optimize locking */
@@ -142,27 +142,27 @@ VCL_Refresh(struct VCL_conf **vcc)
 }
 
 void
-VCL_Ref(struct VCL_conf *vc)
+VCL_Ref(struct vcl *vc)
 {
 
 	Lck_Lock(&vcl_mtx);
-	assert(vc->busy > 0);
-	vc->busy++;
+	assert(vc->conf->busy > 0);
+	vc->conf->busy++;
 	Lck_Unlock(&vcl_mtx);
 }
 
 void
-VCL_Rel(struct VCL_conf **vcc)
+VCL_Rel(struct vcl **vcc)
 {
-	struct VCL_conf *vc;
+	struct vcl *vc;
 
 	AN(*vcc);
 	vc = *vcc;
 	*vcc = NULL;
 
 	Lck_Lock(&vcl_mtx);
-	assert(vc->busy > 0);
-	vc->busy--;
+	assert(vc->conf->busy > 0);
+	vc->conf->busy--;
 	/*
 	 * We do not garbage collect discarded VCL's here, that happens
 	 * in VCL_Poll() which is called from the CLI thread.
@@ -199,7 +199,7 @@ VCL_Open(const char *fn, struct vsb *msg)
 		(void)dlclose(dlh);
 		return (NULL);
 	}
-	ALLOC_OBJ(vcl, VVCLS_MAGIC);
+	ALLOC_OBJ(vcl, VCL_MAGIC);
 	AN(vcl);
 	vcl->dlh = dlh;
 	memcpy(vcl->conf, cnf, sizeof *cnf);
@@ -211,7 +211,7 @@ VCL_Close(struct vcl **vclp)
 {
 	struct vcl *vcl;
 
-	CHECK_OBJ_NOTNULL(*vclp, VVCLS_MAGIC);
+	CHECK_OBJ_NOTNULL(*vclp, VCL_MAGIC);
 	vcl = *vclp;
 	*vclp = NULL;
 	AZ(dlclose(vcl->dlh));
@@ -246,18 +246,18 @@ VCL_TestLoad(const char *fn)
 /*--------------------------------------------------------------------*/
 
 struct director *
-VCL_DefaultDirector(const struct VCL_conf *vcc)
+VCL_DefaultDirector(const struct vcl *vcc)
 {
 
 	AN(vcc);
-	return (*vcc->default_director);
+	return (*vcc->conf->default_director);
 }
 
 const char *
-VCL_Name(const struct VCL_conf *vcc)
+VCL_Name(const struct vcl *vcc)
 {
 	AN(vcc);
-	return (vcc->loaded_name);
+	return (vcc->conf->loaded_name);
 }
 
 /*--------------------------------------------------------------------*/
@@ -570,7 +570,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 		CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC);
 		vsl = req->vsl;
-		ctx.vcl = req->vcl;
+		ctx.vcl = req->vcl->conf;
 		ctx.http_req = req->http;
 		ctx.http_req_top = req->top->http;
 		ctx.http_resp = req->resp;
@@ -581,7 +581,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 	if (bo != NULL) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 		vsl = bo->vsl;
-		ctx.vcl = bo->vcl;
+		ctx.vcl = bo->vcl->conf;
 		ctx.http_bereq = bo->bereq;
 		ctx.http_beresp = bo->beresp;
 		ctx.bo = bo;
@@ -612,14 +612,15 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 
 #define VCL_MET_MAC(func, upper, bitmap)				\
 void									\
-VCL_##func##_method(struct VCL_conf *vcl, struct worker *wrk,		\
+VCL_##func##_method(struct vcl *vcl, struct worker *wrk,		\
      struct req *req, struct busyobj *bo, void *specific)		\
 {									\
 									\
-	CHECK_OBJ_NOTNULL(vcl, VCL_CONF_MAGIC);				\
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);				\
+	CHECK_OBJ_NOTNULL(vcl->conf, VCL_CONF_MAGIC);			\
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);				\
 	vcl_call_method(wrk, req, bo, specific,				\
-	    VCL_MET_ ## upper, vcl->func##_func);			\
+	    VCL_MET_ ## upper, vcl->conf->func##_func);			\
 	AN((1U << wrk->handling) & bitmap);				\
 }
 



From phk at FreeBSD.org  Wed Jun 17 09:39:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 11:39:04 +0200
Subject: [master] c72cfed Make the VCL reference in vrt_ctx point to our
 internal housekeeping rather than the compile config structure.
Message-ID: <E1Z59oG-0000v4-8Y@project.varnish-software.com>


commit c72cfed759692baf3a43f852470846f6e071ccc9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 09:19:15 2015 +0000

    Make the VCL reference in vrt_ctx point to our internal
    housekeeping rather than the compile config structure.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 2df74c2..2720710 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -308,9 +308,9 @@ VRT_init_vbe(VRT_CTX, struct director **dp, const struct vrt_backend *vrt)
 	AN(dp);
 	AZ(*dp);
 	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->vcl, VCL_CONF_MAGIC);
+	AN(ctx->vcl);
 
-	be = VBE_AddBackend(ctx->vcl->loaded_name, vrt);
+	be = VBE_AddBackend(ctx, vrt);
 	AN(be);
 	ALLOC_OBJ(d, DIRECTOR_MAGIC);
 	XXXAN(d);
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index d7c9aac..6c9f82b 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -39,6 +39,7 @@
 
 struct vbp_target;
 struct vbc;
+struct vrt_ctx;
 struct vrt_backend_probe;
 struct tcp_pool;
 
@@ -105,7 +106,8 @@ struct vbc {
 
 /* cache_backend_cfg.c */
 unsigned VBE_Healthy(const struct backend *b, double *changed);
-struct backend *VBE_AddBackend(const char *vcl, const struct vrt_backend *vb);
+struct backend *VBE_AddBackend(const struct vrt_ctx *,
+    const struct vrt_backend *);
 void VBE_DeleteBackend(struct backend *);
 
 /* cache_backend_poll.c */
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 9f831b4..16cc55c 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -76,11 +76,13 @@ VBE_DeleteBackend(struct backend *b)
  */
 
 struct backend *
-VBE_AddBackend(const char *vcl, const struct vrt_backend *vb)
+VBE_AddBackend(const struct vrt_ctx *ctx, const struct vrt_backend *vb)
 {
 	struct backend *b;
 	char buf[128];
+	struct vcl *vcl;
 
+	vcl = ctx->vcl;
 	AN(vb->vcl_name);
 	assert(vb->ipv4_suckaddr != NULL || vb->ipv6_suckaddr != NULL);
 
@@ -89,7 +91,7 @@ VBE_AddBackend(const char *vcl, const struct vrt_backend *vb)
 	XXXAN(b);
 	Lck_New(&b->mtx, lck_backend);
 
-	bprintf(buf, "%s.%s", vcl, vb->vcl_name);
+	bprintf(buf, "%s.%s", VCL_Name(vcl), vb->vcl_name);
 	REPLACE(b->display_name, buf);
 
 	b->vcl_name =  vb->vcl_name;
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 4f1ad1d..0717bcb 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -262,6 +262,20 @@ VCL_Name(const struct vcl *vcc)
 
 /*--------------------------------------------------------------------*/
 
+void
+VRT_count(VRT_CTX, unsigned u)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC);
+	assert(u < ctx->vcl->conf->nref);
+	if (ctx->vsl != NULL)
+		VSLb(ctx->vsl, SLT_VCL_trace, "%u %u.%u", u,
+		    ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos);
+}
+
+/*--------------------------------------------------------------------*/
+
 static struct vcl *
 vcl_find(const char *name)
 {
@@ -332,7 +346,7 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 	INIT_OBJ(&ctx, VRT_CTX_MAGIC);
 	ctx.method = VCL_MET_INIT;
 	ctx.handling = &hand;
-	ctx.vcl = vcl->conf;
+	ctx.vcl = vcl;
 
 	VSB_clear(vsb);
 	ctx.msg = vsb;
@@ -380,7 +394,7 @@ VCL_Nuke(struct vcl *vcl)
 	VTAILQ_REMOVE(&vcl_head, vcl, list);
 	ctx.method = VCL_MET_FINI;
 	ctx.handling = &hand;
-	ctx.vcl = vcl->conf;
+	ctx.vcl = vcl;
 	AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
 	free(vcl->conf->loaded_name);
 	VCL_Close(&vcl);
@@ -569,8 +583,9 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 	if (req != NULL) {
 		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 		CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC);
+		CHECK_OBJ_NOTNULL(req->vcl, VCL_MAGIC);
 		vsl = req->vsl;
-		ctx.vcl = req->vcl->conf;
+		ctx.vcl = req->vcl;
 		ctx.http_req = req->http;
 		ctx.http_req_top = req->top->http;
 		ctx.http_resp = req->resp;
@@ -580,8 +595,9 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 	}
 	if (bo != NULL) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+		CHECK_OBJ_NOTNULL(bo->vcl, VCL_MAGIC);
 		vsl = bo->vsl;
-		ctx.vcl = bo->vcl->conf;
+		ctx.vcl = bo->vcl;
 		ctx.http_bereq = bo->bereq;
 		ctx.http_beresp = bo->beresp;
 		ctx.bo = bo;
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 9b4aaff..df6bb96 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -65,18 +65,6 @@ VRT_error(VRT_CTX, unsigned code, const char *reason)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_count(VRT_CTX, unsigned u)
-{
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	if (ctx->vsl != NULL)
-		VSLb(ctx->vsl, SLT_VCL_trace, "%u %u.%u", u,
-		    ctx->vcl->ref[u].line, ctx->vcl->ref[u].pos);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
 VRT_acl_log(VRT_CTX, const char *msg)
 {
 
diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c
index b4676bc..22b45de 100644
--- a/bin/varnishd/cache/cache_vrt_priv.c
+++ b/bin/varnishd/cache/cache_vrt_priv.c
@@ -43,7 +43,7 @@ struct vrt_priv {
 #define VRT_PRIV_MAGIC			0x24157a52
 	VTAILQ_ENTRY(vrt_priv)		list;
 	struct vmod_priv		priv[1];
-	const struct VCL_conf		*vcl;
+	const struct vcl		*vcl;
 	uintptr_t			id;
 	uintptr_t			vmod_id;
 };
diff --git a/include/vrt.h b/include/vrt.h
index cbed87b..2ac390e 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -49,16 +49,17 @@
 
 /***********************************************************************/
 
-struct req;
+struct VCL_conf;
 struct busyobj;
-struct vsl_log;
-struct http;
-struct ws;
-struct vsb;
 struct director;
-struct VCL_conf;
+struct http;
+struct req;
 struct suckaddr;
+struct vcl;
 struct vmod;
+struct vsb;
+struct vsl_log;
+struct ws;
 
 /***********************************************************************
  * This is the central definition of the mapping from VCL types to
@@ -95,7 +96,7 @@ struct vrt_ctx {
 
 	struct vsb			*msg;	// Only in ...init()
 	struct vsl_log			*vsl;
-	struct VCL_conf			*vcl;
+	struct vcl			*vcl;
 	struct ws			*ws;
 
 	struct req			*req;



From phk at FreeBSD.org  Wed Jun 17 09:39:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 11:39:04 +0200
Subject: [master] ff6d1d8 Move some housekeeping stuff out of VCL_conf and
 into housekeeping struct vcl.
Message-ID: <E1Z59oG-0000vE-9d@project.varnish-software.com>


commit ff6d1d8d764e89c57c810472538121379c0e53c6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 09:30:00 2015 +0000

    Move some housekeeping stuff out of VCL_conf and into housekeeping
    struct vcl.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 0717bcb..b751672 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -51,6 +51,9 @@ struct vcl {
 	void			*dlh;
 	struct VCL_conf		conf[1];
 	char			state[8];
+	char			*loaded_name;
+	unsigned		busy;
+	unsigned		discard;
 	int			warm;
 };
 
@@ -126,8 +129,8 @@ VCL_Get(struct vcl **vcc)
 	AN(vcl_active);
 	*vcc = vcl_active;
 	AN(*vcc);
-	AZ((*vcc)->conf->discard);
-	(*vcc)->conf->busy++;
+	AZ((*vcc)->discard);
+	(*vcc)->busy++;
 	Lck_Unlock(&vcl_mtx);
 }
 
@@ -146,8 +149,8 @@ VCL_Ref(struct vcl *vc)
 {
 
 	Lck_Lock(&vcl_mtx);
-	assert(vc->conf->busy > 0);
-	vc->conf->busy++;
+	assert(vc->busy > 0);
+	vc->busy++;
 	Lck_Unlock(&vcl_mtx);
 }
 
@@ -161,8 +164,8 @@ VCL_Rel(struct vcl **vcc)
 	*vcc = NULL;
 
 	Lck_Lock(&vcl_mtx);
-	assert(vc->conf->busy > 0);
-	vc->conf->busy--;
+	assert(vc->busy > 0);
+	vc->busy--;
 	/*
 	 * We do not garbage collect discarded VCL's here, that happens
 	 * in VCL_Poll() which is called from the CLI thread.
@@ -257,7 +260,7 @@ const char *
 VCL_Name(const struct vcl *vcc)
 {
 	AN(vcc);
-	return (vcc->conf->loaded_name);
+	return (vcc->loaded_name);
 }
 
 /*--------------------------------------------------------------------*/
@@ -283,9 +286,9 @@ vcl_find(const char *name)
 
 	ASSERT_CLI();
 	VTAILQ_FOREACH(vcl, &vcl_head, list) {
-		if (vcl->conf->discard)
+		if (vcl->discard)
 			continue;
-		if (!strcmp(vcl->conf->loaded_name, name))
+		if (!strcmp(vcl->loaded_name, name))
 			return (vcl);
 	}
 	return (NULL);
@@ -340,8 +343,8 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 		return (1);
 	}
 
-	vcl->conf->loaded_name = strdup(name);
-	XXXAN(vcl->conf->loaded_name);
+	vcl->loaded_name = strdup(name);
+	XXXAN(vcl->loaded_name);
 
 	INIT_OBJ(&ctx, VRT_CTX_MAGIC);
 	ctx.method = VCL_MET_INIT;
@@ -389,14 +392,14 @@ VCL_Nuke(struct vcl *vcl)
 	INIT_OBJ(&ctx, VRT_CTX_MAGIC);
 	ASSERT_CLI();
 	assert(vcl != vcl_active);
-	assert(vcl->conf->discard);
-	AZ(vcl->conf->busy);
+	assert(vcl->discard);
+	AZ(vcl->busy);
 	VTAILQ_REMOVE(&vcl_head, vcl, list);
 	ctx.method = VCL_MET_FINI;
 	ctx.handling = &hand;
 	ctx.vcl = vcl;
 	AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
-	free(vcl->conf->loaded_name);
+	free(vcl->loaded_name);
 	VCL_Close(&vcl);
 	VSC_C_main->n_vcl--;
 	VSC_C_main->n_vcl_discard--;
@@ -411,7 +414,7 @@ VCL_Poll(void)
 
 	ASSERT_CLI();
 	VTAILQ_FOREACH_SAFE(vcl, &vcl_head, list, vcl2)
-		if (vcl->conf->discard && vcl->conf->busy == 0)
+		if (vcl->discard && vcl->busy == 0)
 			VCL_Nuke(vcl);
 }
 
@@ -429,15 +432,14 @@ ccf_config_list(struct cli *cli, const char * const *av, void *priv)
 	VTAILQ_FOREACH(vcl, &vcl_head, list) {
 		if (vcl == vcl_active) {
 			flg = "active";
-		} else if (vcl->conf->discard) {
+		} else if (vcl->discard) {
 			flg = "discarded";
 		} else
 			flg = "available";
 		VCLI_Out(cli, "%-10s %4s/%s  %6u %s\n",
 		    flg,
 		    vcl->state, vcl->warm ? "warm" : "cold",
-		    vcl->conf->busy,
-		    vcl->conf->loaded_name);
+		    vcl->busy, vcl->loaded_name);
 	}
 }
 
@@ -481,10 +483,10 @@ ccf_config_discard(struct cli *cli, const char * const *av, void *priv)
 	assert (vcl != vcl_active);	// MGT ensures this
 	VSC_C_main->n_vcl_discard++;
 	VSC_C_main->n_vcl_avail--;
-	vcl->conf->discard = 1;
+	vcl->discard = 1;
 	Lck_Unlock(&vcl_mtx);
 
-	if (vcl->conf->busy == 0)
+	if (vcl->busy == 0)
 		VCL_Nuke(vcl);
 }
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 54560bf..345ae05 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -1058,13 +1058,9 @@ struct VCL_conf {
 	unsigned	magic;
 #define VCL_CONF_MAGIC	0x7406c509	/* from /dev/random */
 
-	char		*loaded_name;
-
 	struct director	**default_director;
-	struct vrt_ref	*ref;
 	unsigned	nref;
-	unsigned	busy;
-	unsigned	discard;
+	struct vrt_ref	*ref;
 
 	unsigned	nsrc;
 	const char	**srcname;



From phk at FreeBSD.org  Wed Jun 17 09:39:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 11:39:04 +0200
Subject: [master] 97be245 Get rid of the unused "count" field in vrt_ref.
Message-ID: <E1Z59oG-0000vO-Ad@project.varnish-software.com>


commit 97be2458dc680490187a29e380113c61ce2e6f68
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 09:37:56 2015 +0000

    Get rid of the unused "count" field in vrt_ref.

diff --git a/include/vrt.h b/include/vrt.h
index 2ac390e..bd816e0 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -199,7 +199,6 @@ struct vrt_ref {
 	unsigned	offset;
 	unsigned	line;
 	unsigned	pos;
-	unsigned	count;
 	const char	*token;
 };
 
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index f21045b..511f20e 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -312,7 +312,7 @@ EmitCoordinates(const struct vcc *tl, struct vsb *vsb)
 				pos++;
 
 		}
-		VSB_printf(vsb, "  [%3u] = { %u, %8tu, %4u, %3u, 0, ",
+		VSB_printf(vsb, "  [%3u] = { %u, %8tu, %4u, %3u, ",
 		    t->cnt, sp->idx, t->b - sp->b, lin, pos + 1);
 		if (t->tok == CSRC)
 			VSB_printf(vsb, " \"C{\"},\n");



From phk at FreeBSD.org  Wed Jun 17 09:39:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 11:39:04 +0200
Subject: [master] e38f702 No need to copy the VCL_conf struct anymore,
 it can be const.
Message-ID: <E1Z59oG-0000vX-Bg@project.varnish-software.com>


commit e38f70269013cfaf53189f7addcf89860036ab72
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 09:38:16 2015 +0000

    No need to copy the VCL_conf struct anymore, it can be const.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index b751672..b5fd367 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -49,7 +49,7 @@ struct vcl {
 #define VCL_MAGIC		0x214188f2
 	VTAILQ_ENTRY(vcl)	list;
 	void			*dlh;
-	struct VCL_conf		conf[1];
+	const struct VCL_conf	*conf;
 	char			state[8];
 	char			*loaded_name;
 	unsigned		busy;
@@ -205,7 +205,7 @@ VCL_Open(const char *fn, struct vsb *msg)
 	ALLOC_OBJ(vcl, VCL_MAGIC);
 	AN(vcl);
 	vcl->dlh = dlh;
-	memcpy(vcl->conf, cnf, sizeof *cnf);
+	vcl->conf = cnf;
 	return (vcl);
 }
 



From dridi.boukelmoune at gmail.com  Wed Jun 17 12:31:17 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 17 Jun 2015 14:31:17 +0200
Subject: [master] f9b6914 Gracefully fail shell commands in varnishtest
Message-ID: <E1Z5CUv-0008Mx-Gt@project.varnish-software.com>


commit f9b6914410edd787925a14d22351cdb90063e8cd
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jun 17 14:29:27 2015 +0200

    Gracefully fail shell commands in varnishtest

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index ae38f68..3af21e6 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -428,7 +428,7 @@ cmd_shell(CMD_ARGS)
 {
 	(void)priv;
 	(void)cmd;
-	int r;
+	int r, s;
 
 	if (av == NULL)
 		return;
@@ -436,7 +436,11 @@ cmd_shell(CMD_ARGS)
 	AZ(av[2]);
 	vtc_dump(vl, 4, "shell", av[1], -1);
 	r = system(av[1]);
-	AZ(WEXITSTATUS(r));
+	s = WEXITSTATUS(r);
+	if (s != 0) {
+		vtc_log(vl, 0, "CMD '%s' failed with status %d",
+		    av[1], s);
+	}
 }
 
 /**********************************************************************



From dridi.boukelmoune at gmail.com  Wed Jun 17 12:34:45 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 17 Jun 2015 14:34:45 +0200
Subject: [master] 02f1654 No need for curly braces
Message-ID: <E1Z5CYH-0000Ds-J8@project.varnish-software.com>


commit 02f1654dab6300faca5972c39dc0532f752bd1bb
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jun 17 14:32:44 2015 +0200

    No need for curly braces

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 3af21e6..f44d220 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -437,10 +437,9 @@ cmd_shell(CMD_ARGS)
 	vtc_dump(vl, 4, "shell", av[1], -1);
 	r = system(av[1]);
 	s = WEXITSTATUS(r);
-	if (s != 0) {
+	if (s != 0)
 		vtc_log(vl, 0, "CMD '%s' failed with status %d",
 		    av[1], s);
-	}
 }
 
 /**********************************************************************



From phk at FreeBSD.org  Wed Jun 17 12:44:52 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 14:44:52 +0200
Subject: [master] 1b4695e Rename VRT_init_vbe() to VRT_new_backend()
Message-ID: <E1Z5Ci4-0001RU-26@project.varnish-software.com>


commit 1b4695eb9b2f8680be0fdd68652a6cf08d88cba0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 12:03:32 2015 +0000

    Rename VRT_init_vbe() to VRT_new_backend()

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 2720710..c029e44 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -297,16 +297,14 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 
 /*--------------------------------------------------------------------*/
 
-void
-VRT_init_vbe(VRT_CTX, struct director **dp, const struct vrt_backend *vrt)
+struct director *
+VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 {
 	struct director *d;
 	struct backend *be;
 
 	ASSERT_CLI();
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	AN(dp);
-	AZ(*dp);
 	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
 	AN(ctx->vcl);
 
@@ -328,7 +326,7 @@ VRT_init_vbe(VRT_CTX, struct director **dp, const struct vrt_backend *vrt)
 	if (vrt->probe != NULL)
 		VBP_Insert(be, vrt->probe, vrt->hosthdr);
 
-	*dp = d;
+	return (d);
 }
 
 void
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 6c9f82b..b586d22 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -62,6 +62,7 @@ struct backend {
 	int			refcount;
 	struct lock		mtx;
 
+	const struct vcl	*vcl;
 	const char		*vcl_name;
 	char			*display_name;
 	const char		*ipv4_addr;
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 16cc55c..b9c5591 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -82,7 +82,9 @@ VBE_AddBackend(const struct vrt_ctx *ctx, const struct vrt_backend *vb)
 	char buf[128];
 	struct vcl *vcl;
 
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	vcl = ctx->vcl;
+	AN(vcl);
 	AN(vb->vcl_name);
 	assert(vb->ipv4_suckaddr != NULL || vb->ipv6_suckaddr != NULL);
 
@@ -94,6 +96,7 @@ VBE_AddBackend(const struct vrt_ctx *ctx, const struct vrt_backend *vb)
 	bprintf(buf, "%s.%s", VCL_Name(vcl), vb->vcl_name);
 	REPLACE(b->display_name, buf);
 
+	b->vcl = vcl;
 	b->vcl_name =  vb->vcl_name;
 	b->ipv4_addr = vb->ipv4_addr;
 	b->ipv6_addr = vb->ipv6_addr;
diff --git a/include/vrt.h b/include/vrt.h
index bd816e0..5afcf33 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -242,7 +242,7 @@ void VRT_Rollback(VRT_CTX, const struct http *);
 void VRT_synth_page(VRT_CTX, const char *, ...);
 
 /* Backend related */
-void VRT_init_vbe(VRT_CTX, struct director **, const struct vrt_backend *);
+struct director *VRT_new_backend(VRT_CTX, const struct vrt_backend *);
 #ifdef VCL_RET_MAX
 void VRT_event_vbe(VRT_CTX, enum vcl_event_e, const struct director *,
     const struct vrt_backend *);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 59fb851..c1d96ac 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -410,7 +410,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 
 	ifp = New_IniFin(tl);
 	VSB_printf(ifp->ini,
-	    "\tVRT_init_vbe(ctx, &%s,\n\t    &vgc_dir_priv_%s);",
+	    "\t%s =\n\t    VRT_new_backend(ctx, &vgc_dir_priv_%s);",
 	    vgcname, vgcname);
 	VSB_printf(ifp->fin,
 	    "\t\tVRT_fini_vbe(ctx, &%s,\n\t\t    &vgc_dir_priv_%s);",



From phk at FreeBSD.org  Wed Jun 17 12:44:52 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 14:44:52 +0200
Subject: [master] 28bdbcc Rename VRT_fini_vbe() to VRT_delete_backen()
Message-ID: <E1Z5Ci4-0001Ra-3M@project.varnish-software.com>


commit 28bdbcc2ba262ae9c1d7934e3049c7f8b392eaad
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 12:20:27 2015 +0000

    Rename VRT_fini_vbe() to VRT_delete_backen()

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index c029e44..24b1e07 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -362,7 +362,7 @@ VRT_event_vbe(VRT_CTX, enum vcl_event_e ev, const struct director *d,
 }
 
 void
-VRT_fini_vbe(VRT_CTX, struct director **dp, const struct vrt_backend *vrt)
+VRT_delete_backend(VRT_CTX, struct director **dp)
 {
 	struct director *d;
 	struct backend *be;
@@ -371,15 +371,13 @@ VRT_fini_vbe(VRT_CTX, struct director **dp, const struct vrt_backend *vrt)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(dp);
 	AN(*dp);
-	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
 
 	d = *dp;
 	*dp = NULL;
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	assert(d->priv2 == vrt);
 	CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC);
 
-	if (vrt->probe != NULL)
+	if (be->probe != NULL)
 		VBP_Remove(be);
 
 	VBE_DeleteBackend(be);
diff --git a/include/vrt.h b/include/vrt.h
index 5afcf33..27e738e 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -247,7 +247,7 @@ struct director *VRT_new_backend(VRT_CTX, const struct vrt_backend *);
 void VRT_event_vbe(VRT_CTX, enum vcl_event_e, const struct director *,
     const struct vrt_backend *);
 #endif
-void VRT_fini_vbe(VRT_CTX, struct director **, const struct vrt_backend *);
+void VRT_delete_backend(VRT_CTX, struct director **);
 
 /* Suckaddr related */
 int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index c1d96ac..2bb3693 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -413,8 +413,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 	    "\t%s =\n\t    VRT_new_backend(ctx, &vgc_dir_priv_%s);",
 	    vgcname, vgcname);
 	VSB_printf(ifp->fin,
-	    "\t\tVRT_fini_vbe(ctx, &%s,\n\t\t    &vgc_dir_priv_%s);",
-	    vgcname, vgcname);
+	    "\t\tVRT_delete_backend(ctx, &%s);\n", vgcname);
 	VSB_printf(ifp->event,
 	    "\tVRT_event_vbe(ctx, ev, %s, &vgc_dir_priv_%s);",
 	    vgcname, vgcname);



From phk at FreeBSD.org  Wed Jun 17 12:44:52 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jun 2015 14:44:52 +0200
Subject: [master] cc70c58 Rearrange the backend code a bit.
Message-ID: <E1Z5Ci4-0001Rj-4W@project.varnish-software.com>


commit cc70c5807df0d5548a39148dc672755375853db9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 17 12:44:37 2015 +0000

    Rearrange the backend code a bit.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 49bf6ba..2b025ef 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -678,6 +678,9 @@ void VCA_Shutdown(void);
 /* cache_backend_cfg.c */
 void VBE_InitCfg(void);
 
+/* cache_backend_poll.c */
+void VBP_Init(void);
+
 /* cache_ban.c */
 struct ban *BAN_New(void);
 int BAN_AddTest(struct ban *, const char *, const char *, const char *);
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 24b1e07..930598e 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -36,9 +36,8 @@
 
 #include "cache.h"
 
-#include "cache_backend.h"
 #include "cache_director.h"
-#include "vcl.h"
+#include "cache_backend.h"
 #include "vrt.h"
 #include "vtcp.h"
 
@@ -297,21 +296,16 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 
 /*--------------------------------------------------------------------*/
 
-struct director *
-VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
+void
+VBE_fill_director(struct backend *be, const struct vrt_backend *vrt)
 {
 	struct director *d;
-	struct backend *be;
 
-	ASSERT_CLI();
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
 	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
-	AN(ctx->vcl);
 
-	be = VBE_AddBackend(ctx, vrt);
-	AN(be);
-	ALLOC_OBJ(d, DIRECTOR_MAGIC);
-	XXXAN(d);
+	INIT_OBJ(be->director, DIRECTOR_MAGIC);
+	d = be->director;
 	d->priv = be;
 	d->priv2 = vrt;
 	d->name = "backend";
@@ -322,65 +316,4 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 	d->getbody = vbe_dir_getbody;
 	d->getip = vbe_dir_getip;
 	d->finish = vbe_dir_finish;
-
-	if (vrt->probe != NULL)
-		VBP_Insert(be, vrt->probe, vrt->hosthdr);
-
-	return (d);
-}
-
-void
-VRT_event_vbe(VRT_CTX, enum vcl_event_e ev, const struct director *d,
-    const struct vrt_backend *vrt)
-{
-	struct backend *be;
-
-	ASSERT_CLI();
-	(void)ev;
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
-	assert(d->priv2 == vrt);
-
-	CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC);
-	if (ev == VCL_EVENT_WARM) {
-		be->vsc = VSM_Alloc(sizeof *be->vsc,
-		    VSC_CLASS, VSC_type_vbe, be->display_name);
-		AN(be->vsc);
-	}
-
-	if (be->probe != NULL && ev == VCL_EVENT_WARM)
-		VBP_Control(be, 1);
-
-	if (be->probe != NULL && ev == VCL_EVENT_COLD)
-		VBP_Control(be, 0);
-
-	if (ev == VCL_EVENT_COLD) {
-		VSM_Free(be->vsc);
-		be->vsc = NULL;
-	}
-}
-
-void
-VRT_delete_backend(VRT_CTX, struct director **dp)
-{
-	struct director *d;
-	struct backend *be;
-
-	ASSERT_CLI();
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	AN(dp);
-	AN(*dp);
-
-	d = *dp;
-	*dp = NULL;
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC);
-
-	if (be->probe != NULL)
-		VBP_Remove(be);
-
-	VBE_DeleteBackend(be);
-	free(d->vcl_name);
-	FREE_OBJ(d);
 }
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index b586d22..c3b9152 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -82,6 +82,8 @@ struct backend {
 	struct VSC_C_vbe	*vsc;
 
 	struct tcp_pool		*tcp_pool;
+
+	struct director		director[1];
 };
 
 /* -------------------------------------------------------------------*/
@@ -105,11 +107,11 @@ struct vbc {
 	struct worker		*wrk;
 };
 
+/* cache_backend.c */
+void VBE_fill_director(struct backend *be, const struct vrt_backend *vrt);
+
 /* cache_backend_cfg.c */
 unsigned VBE_Healthy(const struct backend *b, double *changed);
-struct backend *VBE_AddBackend(const struct vrt_ctx *,
-    const struct vrt_backend *);
-void VBE_DeleteBackend(struct backend *);
 
 /* cache_backend_poll.c */
 void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p,
@@ -117,7 +119,6 @@ void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p,
 void VBP_Remove(struct backend *b);
 void VBP_Control(const struct backend *b, int stop);
 void VBP_Status(struct cli *cli, const struct backend *, int details);
-void VBP_Init(void);
 
 struct tcp_pool *VBT_Ref(const struct suckaddr *ip4,
     const struct suckaddr *ip6);
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index b9c5591..27137d8 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -38,76 +38,56 @@
 
 #include "cache.h"
 
+#include "cache_director.h"
 #include "cache_backend.h"
 #include "vcli.h"
 #include "vcli_priv.h"
 #include "vsa.h"
+#include "vcl.h"
 #include "vrt.h"
 #include "vtim.h"
 
 static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends);
 static struct lock backends_mtx;
-
-/*--------------------------------------------------------------------
- */
-
-void
-VBE_DeleteBackend(struct backend *b)
-{
-
-	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-
-	Lck_Lock(&backends_mtx);
-	VTAILQ_REMOVE(&backends, b, list);
-	VSC_C_main->n_backend--;
-	Lck_Unlock(&backends_mtx);
-
-	free(b->ipv4);
-	free(b->ipv6);
-	free(b->display_name);
-	AZ(b->vsc);
-	VBT_Rel(&b->tcp_pool);
-	Lck_Delete(&b->mtx);
-	FREE_OBJ(b);
-}
-
 /*--------------------------------------------------------------------
  * Create a new director::backend instance.
  */
 
-struct backend *
-VBE_AddBackend(const struct vrt_ctx *ctx, const struct vrt_backend *vb)
+struct director *
+VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 {
 	struct backend *b;
 	char buf[128];
 	struct vcl *vcl;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
+
 	vcl = ctx->vcl;
 	AN(vcl);
-	AN(vb->vcl_name);
-	assert(vb->ipv4_suckaddr != NULL || vb->ipv6_suckaddr != NULL);
+	AN(vrt->vcl_name);
+	assert(vrt->ipv4_suckaddr != NULL || vrt->ipv6_suckaddr != NULL);
 
 	/* Create new backend */
 	ALLOC_OBJ(b, BACKEND_MAGIC);
 	XXXAN(b);
 	Lck_New(&b->mtx, lck_backend);
 
-	bprintf(buf, "%s.%s", VCL_Name(vcl), vb->vcl_name);
+	bprintf(buf, "%s.%s", VCL_Name(vcl), vrt->vcl_name);
 	REPLACE(b->display_name, buf);
 
 	b->vcl = vcl;
-	b->vcl_name =  vb->vcl_name;
-	b->ipv4_addr = vb->ipv4_addr;
-	b->ipv6_addr = vb->ipv6_addr;
-	b->port = vb->port;
+	b->vcl_name =  vrt->vcl_name;
+	b->ipv4_addr = vrt->ipv4_addr;
+	b->ipv6_addr = vrt->ipv6_addr;
+	b->port = vrt->port;
 
-	b->tcp_pool = VBT_Ref(vb->ipv4_suckaddr, vb->ipv6_suckaddr);
+	b->tcp_pool = VBT_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr);
 
-	if (vb->ipv4_suckaddr != NULL)
-		b->ipv4 = VSA_Clone(vb->ipv4_suckaddr);
-	if (vb->ipv6_suckaddr != NULL)
-		b->ipv6 = VSA_Clone(vb->ipv6_suckaddr);
+	if (vrt->ipv4_suckaddr != NULL)
+		b->ipv4 = VSA_Clone(vrt->ipv4_suckaddr);
+	if (vrt->ipv6_suckaddr != NULL)
+		b->ipv6 = VSA_Clone(vrt->ipv6_suckaddr);
 
 	assert(b->ipv4 != NULL || b->ipv6 != NULL);
 
@@ -119,7 +99,80 @@ VBE_AddBackend(const struct vrt_ctx *ctx, const struct vrt_backend *vb)
 	VTAILQ_INSERT_TAIL(&backends, b, list);
 	VSC_C_main->n_backend++;
 	Lck_Unlock(&backends_mtx);
-	return (b);
+
+	VBE_fill_director(b, vrt);
+
+	if (vrt->probe != NULL)
+		VBP_Insert(b, vrt->probe, vrt->hosthdr);
+
+	return (b->director);
+}
+
+void
+VRT_event_vbe(VRT_CTX, enum vcl_event_e ev, const struct director *d,
+    const struct vrt_backend *vrt)
+{
+	struct backend *be;
+
+	ASSERT_CLI();
+	(void)ev;
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
+	assert(d->priv2 == vrt);
+
+	CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC);
+	if (ev == VCL_EVENT_WARM) {
+		be->vsc = VSM_Alloc(sizeof *be->vsc,
+		    VSC_CLASS, VSC_type_vbe, be->display_name);
+		AN(be->vsc);
+	}
+
+	if (be->probe != NULL && ev == VCL_EVENT_WARM)
+		VBP_Control(be, 1);
+
+	if (be->probe != NULL && ev == VCL_EVENT_COLD)
+		VBP_Control(be, 0);
+
+	if (ev == VCL_EVENT_COLD) {
+		VSM_Free(be->vsc);
+		be->vsc = NULL;
+	}
+}
+
+void
+VRT_delete_backend(VRT_CTX, struct director **dp)
+{
+	struct director *d;
+	struct backend *be;
+
+	ASSERT_CLI();
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(dp);
+	AN(*dp);
+
+	d = *dp;
+	*dp = NULL;
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC);
+
+	if (be->probe != NULL)
+		VBP_Remove(be);
+
+	free(d->vcl_name);
+
+	Lck_Lock(&backends_mtx);
+	VTAILQ_REMOVE(&backends, be, list);
+	VSC_C_main->n_backend--;
+	Lck_Unlock(&backends_mtx);
+
+	free(be->ipv4);
+	free(be->ipv6);
+	free(be->display_name);
+	AZ(be->vsc);
+	VBT_Rel(&be->tcp_pool);
+	Lck_Delete(&be->mtx);
+	FREE_OBJ(be);
 }
 
 /*---------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index c658417..fe6d735 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -45,6 +45,7 @@
 
 #include "cache.h"
 
+#include "cache_director.h"
 #include "cache_backend.h"
 #include "vcli_priv.h"
 #include "vrt.h"
diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index 70afa2e..c830fa5 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -38,6 +38,7 @@
 
 #include "cache.h"
 
+#include "cache_director.h"
 #include "cache_backend.h"
 #include "cache_pool.h"
 #include "vtcp.h"
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 4c5370f..da9c297 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -33,7 +33,6 @@
 #include <stdlib.h>
 
 #include "cache.h"
-#include "cache_backend.h"
 #include "common/heritage.h"
 
 #include "vcli_priv.h"
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 3755508..b959ff6 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -43,6 +43,7 @@
 #include "cache_filter.h"
 #include "common/heritage.h"
 
+#include "cache_director.h"
 #include "cache_backend.h"
 #include "storage/storage.h"
 #include "vcli_priv.h"
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 23fc6b9..1eae475 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -37,8 +37,8 @@
 
 #include "hash/hash_slinger.h"
 
-#include "cache/cache_backend.h"
 #include "cache/cache_director.h"
+#include "cache/cache_backend.h"
 #include "vcli_priv.h"
 #include "vtcp.h"
 #include "vtim.h"
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index 818f88f..bf7ab67 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -36,8 +36,8 @@
 
 #include "cache/cache.h"
 
-#include "cache/cache_backend.h"
 #include "cache/cache_director.h"
+#include "cache/cache_backend.h"
 #include "vtcp.h"
 #include "vtim.h"
 



From dridi.boukelmoune at gmail.com  Wed Jun 17 14:08:39 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 17 Jun 2015 16:08:39 +0200
Subject: [master] a66ec75 Give more delay u00001.vtc for slower architectures
Message-ID: <E1Z5E19-00025c-Eh@project.varnish-software.com>


commit a66ec75dcb41a8dfd9b2e241e33d8ea24a6ac0c5
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jun 17 16:04:53 2015 +0200

    Give more delay u00001.vtc for slower architectures

diff --git a/bin/varnishtest/tests/u00001.vtc b/bin/varnishtest/tests/u00001.vtc
index f362a0c..8f35294 100644
--- a/bin/varnishtest/tests/u00001.vtc
+++ b/bin/varnishtest/tests/u00001.vtc
@@ -15,7 +15,7 @@ varnish v1 -vcl+backend "" -start
 process p1 "exec ${varnishncsa} -n ${v1_name} -w ${tmpdir}/ncsa.log -F %s" -start
 
 # give varnishncsa enough time to open the VSM
-delay 1
+delay 2
 
 client c1 {
 	txreq -url "/foo"
@@ -23,7 +23,7 @@ client c1 {
 } -run
 
 # give varnishncsa enough time to write
-delay 1
+delay 2
 
 # rotate logs
 shell "mv ${tmpdir}/ncsa.log ${tmpdir}/ncsa.old.log >/dev/null 2>&1"
@@ -35,7 +35,7 @@ client c1 {
 } -run
 
 # give varnishncsa enough time to write
-delay 1
+delay 2
 
 process p1 -stop
 



From fgsch at lodoss.net  Thu Jun 18 13:20:28 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 18 Jun 2015 15:20:28 +0200
Subject: [master] 3e5700b It's possible to return abandon from v_b_e{} now
Message-ID: <E1Z5Zk4-0002Xo-8s@project.varnish-software.com>


commit 3e5700b50f62ff79f7677383d5e45f79031403ee
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Jun 18 14:20:06 2015 +0100

    It's possible to return abandon from v_b_e{} now

diff --git a/doc/graphviz/cache_fetch.dot b/doc/graphviz/cache_fetch.dot
index 4bc64de..d477a7b 100644
--- a/doc/graphviz/cache_fetch.dot
+++ b/doc/graphviz/cache_fetch.dot
@@ -79,7 +79,7 @@ digraph cache_fetch {
 		/* vbf_stp_error */
 		v_b_e [
 			    shape=record
-			    label="{vbf_stp_error:|{vcl_backend_error\{\}|{bereq.*|beresp.*}}|{{retry|{<max>max?|<retry>ok?}}|<deliver>deliver}}}"
+			    label="{vbf_stp_error:|{vcl_backend_error\{\}|{bereq.*|beresp.*}}|{{retry|{<max>max?|<retry>ok?}}|abandon|<deliver>deliver}}}"
 			    ]
 		// v_b_e:deliver aka "backend synth" - goes into cache
 		v_b_e:deliver -> FETCH_DONE [label="\"backend synth\""]
@@ -98,4 +98,4 @@ digraph cache_fetch {
 		FETCH_FAIL [label="FETCH_FAIL",
 			    shape=box,style=filled,color=turquoise]
 	}
-}
\ No newline at end of file
+}



From lkarsten at varnish-software.com  Thu Jun 18 14:32:00 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 18 Jun 2015 16:32:00 +0200
Subject: [4.0] 9b22a01 Update changes file with commits since 4.0.3.
Message-ID: <E1Z5arI-00035N-Bh@project.varnish-software.com>


commit 9b22a01be83d54c45ecdb5ea810826442c7c3a98
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Jun 18 16:31:27 2015 +0200

    Update changes file with commits since 4.0.3.

diff --git a/doc/changes.rst b/doc/changes.rst
index bedf36a..f354aea 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -1,3 +1,30 @@
+
+==================================
+Changes from 4.0.3 to (unreleased)
+==================================
+
+* Percent-based storage size configuration was broken
+  and have been retired.
+* Requests with multiple Content-Length headers are now dropped.
+* Python is now a dependency of varnish-libs-devel on Redhat systems. (vmodtool.py)
+* varnishreplay documentation have been retired.
+
+Bugs fixed
+----------
+
+.. _1744: https://www.varnish-cache.org/trac/ticket/1744
+.. _1742: https://www.varnish-cache.org/trac/ticket/1742
+.. _1691: https://www.varnish-cache.org/trac/ticket/1691
+.. _1688: https://www.varnish-cache.org/trac/ticket/1688
+.. _1602: https://www.varnish-cache.org/trac/ticket/1602
+
+- 1744_ - Update the users guide to for new -sfile syntax
+- 1742_ - Document varnishlog -w/-r with more details
+- 1691_ - Fail fetch on malformed Content-Length header
+- 1688_ - Add a VDP_pretend_gzip for use with synth bodies in ESI includes with gzip
+- 1602_ - Deal with known zero length objects properly when handling do_gzip/do_gunzip
+
+
 ============================================
 Changes from 4.0.3-rc3 to 4.0.3 (2015-02-17)
 ============================================



From tfheen at err.no  Thu Jun 18 14:56:29 2015
From: tfheen at err.no (Tollef Fog Heen)
Date: Thu, 18 Jun 2015 16:56:29 +0200
Subject: [master] 327346d Avoid spurious logrotate mail under systemd
Message-ID: <E1Z5bEz-0007ie-Cy@project.varnish-software.com>


commit 327346d9284f2739f1818e35a66a8a7af48fcf0e
Author: Tollef Fog Heen <tfheen at err.no>
Date:   Thu Jun 18 16:55:33 2015 +0200

    Avoid spurious logrotate mail under systemd
    
    Ensure varnishlog/varnishncsa are running before trying to reload
    them, to avoid a spurious mail.

diff --git a/changelog b/changelog
index 37af55f..4104c81 100644
--- a/changelog
+++ b/changelog
@@ -1,3 +1,10 @@
+varnish (4.0.3-2) UNRELEASED; urgency=medium
+
+  * Under systemd, ensure varnishlog/varnishncsa are running before trying
+    to reload them, to avoid a spurious mail.
+
+ -- Tollef Fog Heen <tfheen at debian.org>  Fri, 12 Jun 2015 09:46:22 +0200
+
 varnish (3.0.2-2~bpo60+1) squeeze-backports; urgency=low
 
   * Rebuild for squeeze-backports.
diff --git a/varnish.logrotate b/varnish.logrotate
index 010abe0..b5664a8 100644
--- a/varnish.logrotate
+++ b/varnish.logrotate
@@ -6,6 +6,9 @@
   delaycompress
   missingok
   postrotate
+    if [ -d /run/systemd/system ]; then
+       systemctl -q is-active varnishlog.service || exit 0
+    fi
     /usr/sbin/invoke-rc.d varnishlog reload > /dev/null
   endscript
 }
@@ -18,6 +21,9 @@
   delaycompress
   missingok
   postrotate
+    if [ -d /run/systemd/system ]; then
+       systemctl -q is-active varnishlog.service || exit 0
+    fi
     /usr/sbin/invoke-rc.d varnishncsa reload > /dev/null
   endscript
 }



From lkarsten at varnish-software.com  Thu Jun 18 15:00:26 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 18 Jun 2015 17:00:26 +0200
Subject: [master] 67dd349 Reserve room in the name scheme for private tests.
Message-ID: <E1Z5bIo-000851-Ak@project.varnish-software.com>


commit 67dd349b33d1cb565dcde9e979817e1c17e714b4
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Jun 18 16:38:31 2015 +0200

    Reserve room in the name scheme for private tests.
    
    Duplicate b00043.vtc was no fun.
    
    Lacking a better idea, I'm moving all local tests up 10000
    units, so our local test case is now b10043.vtc.

diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README
index 52e5061..85b309b 100644
--- a/bin/varnishtest/tests/README
+++ b/bin/varnishtest/tests/README
@@ -29,3 +29,15 @@ Naming scheme
 	id ~ [t] --> sTreaming tests
 	id ~ [u] --> Unusual background processes
 	id ~ [v] --> VCL tests: execute VRT functions
+
+
+Private test scope
+------------------
+
+	Test cases matching
+
+		[id]1%04d.vtc
+
+	are reserved for private tests.
+
+



From lkarsten at varnish-software.com  Fri Jun 19 14:12:03 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 19 Jun 2015 16:12:03 +0200
Subject: [4.0] 5cc3f04 s/have/has/
Message-ID: <E1Z5x1X-0008OX-BI@project.varnish-software.com>


commit 5cc3f0419e926ab4bd086424132731c3ed178732
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Jun 19 16:11:47 2015 +0200

    s/have/has/

diff --git a/doc/changes.rst b/doc/changes.rst
index f354aea..d92ce53 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -4,7 +4,7 @@ Changes from 4.0.3 to (unreleased)
 ==================================
 
 * Percent-based storage size configuration was broken
-  and have been retired.
+  and has been retired.
 * Requests with multiple Content-Length headers are now dropped.
 * Python is now a dependency of varnish-libs-devel on Redhat systems. (vmodtool.py)
 * varnishreplay documentation have been retired.



From dridi.boukelmoune at gmail.com  Fri Jun 19 14:15:50 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 19 Jun 2015 16:15:50 +0200
Subject: [4.0] 2aa3546 s/have/has/
Message-ID: <E1Z5x5C-0002cd-V7@project.varnish-software.com>


commit 2aa35465df53fa95f113e57b471b3cff8f496474
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jun 19 16:14:03 2015 +0200

    s/have/has/

diff --git a/doc/changes.rst b/doc/changes.rst
index d92ce53..2ecd3af 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -7,7 +7,7 @@ Changes from 4.0.3 to (unreleased)
   and has been retired.
 * Requests with multiple Content-Length headers are now dropped.
 * Python is now a dependency of varnish-libs-devel on Redhat systems. (vmodtool.py)
-* varnishreplay documentation have been retired.
+* varnishreplay documentation has been retired.
 
 Bugs fixed
 ----------



From phk at FreeBSD.org  Mon Jun 22 09:39:11 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Jun 2015 11:39:11 +0200
Subject: [master] d00f35a Remember to release the VCL.
Message-ID: <E1Z6yC7-0002rl-BL@project.varnish-software.com>


commit d00f35ac13911592b7f27c48fa72da0fd7e1d476
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 22 08:39:34 2015 +0000

    Remember to release the VCL.

diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 27137d8..e2350e2 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -241,6 +241,7 @@ backend_find(struct cli *cli, const char *matcher, bf_func *func, void *priv)
 	}
 	Lck_Unlock(&backends_mtx);
 	VSB_delete(vsb);
+	VCL_Rel(&vcc);
 	return (found);
 }
 



From phk at FreeBSD.org  Mon Jun 22 09:39:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Jun 2015 11:39:12 +0200
Subject: [master] 8bb3709 Assert that we have stats counters when we try to
 get a fd from a backend. (ie: the VCL is warm)
Message-ID: <E1Z6yC8-0002rt-CT@project.varnish-software.com>


commit 8bb370944142ac8900e71087a26f0bbbbd91359b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 22 08:53:21 2015 +0000

    Assert that we have stats counters when we try to get a fd from a backend.
    (ie: the VCL is warm)

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 930598e..a057a14 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -90,6 +90,7 @@ vbe_dir_getfd(struct worker *wrk, const struct director *d, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
+	AN(bp->vsc);
 	CAST_OBJ_NOTNULL(vrt, d->priv2, VRT_BACKEND_MAGIC);
 
 	if (!VBE_Healthy(bp, NULL)) {



From phk at FreeBSD.org  Mon Jun 22 09:39:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Jun 2015 11:39:12 +0200
Subject: [master] c759e55 Add a transient "cooling" state to VCLs,
 while we wait for the busy count to drop to zero.
Message-ID: <E1Z6yC8-0002s5-ED@project.varnish-software.com>


commit c759e55394d833616e2cf9ed6d0318536e67ccad
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 22 09:38:28 2015 +0000

    Add a transient "cooling" state to VCLs, while we wait for the
    busy count to drop to zero.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index b5fd367..a6d7e80 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -44,6 +44,10 @@
 #include "vcli.h"
 #include "vcli_priv.h"
 
+static const char * const vcl_temp_cold = "cold";
+static const char * const vcl_temp_warm = "warn";
+static const char * const vcl_temp_cooling = "cooling";
+
 struct vcl {
 	unsigned		magic;
 #define VCL_MAGIC		0x214188f2
@@ -54,7 +58,7 @@ struct vcl {
 	char			*loaded_name;
 	unsigned		busy;
 	unsigned		discard;
-	int			warm;
+	const char		*temp;
 };
 
 /*
@@ -118,13 +122,11 @@ VCL_Method_Name(unsigned m)
 static void
 VCL_Get(struct vcl **vcc)
 {
-	static int once = 0;
-
-	while (!once && vcl_active == NULL) {
-		(void)sleep(1);
-	}
-	once = 1;
+	while (vcl_active == NULL)
+		(void)usleep(100000);
 
+	CHECK_OBJ_NOTNULL(vcl_active, VCL_MAGIC);
+	assert(vcl_active->temp == vcl_temp_warm);
 	Lck_Lock(&vcl_mtx);
 	AN(vcl_active);
 	*vcc = vcl_active;
@@ -137,6 +139,8 @@ VCL_Get(struct vcl **vcc)
 void
 VCL_Refresh(struct vcl **vcc)
 {
+	CHECK_OBJ_NOTNULL(vcl_active, VCL_MAGIC);
+	assert(vcl_active->temp == vcl_temp_warm);
 	if (*vcc == vcl_active)
 		return;
 	if (*vcc != NULL)
@@ -145,27 +149,30 @@ VCL_Refresh(struct vcl **vcc)
 }
 
 void
-VCL_Ref(struct vcl *vc)
+VCL_Ref(struct vcl *vcl)
 {
 
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
+	assert(vcl->temp == vcl_temp_warm);
 	Lck_Lock(&vcl_mtx);
-	assert(vc->busy > 0);
-	vc->busy++;
+	assert(vcl->busy > 0);
+	vcl->busy++;
 	Lck_Unlock(&vcl_mtx);
 }
 
 void
 VCL_Rel(struct vcl **vcc)
 {
-	struct vcl *vc;
+	struct vcl *vcl;
 
 	AN(*vcc);
-	vc = *vcc;
+	vcl = *vcc;
 	*vcc = NULL;
 
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
 	Lck_Lock(&vcl_mtx);
-	assert(vc->busy > 0);
-	vc->busy--;
+	assert(vcl->busy > 0);
+	vcl->busy--;
 	/*
 	 * We do not garbage collect discarded VCL's here, that happens
 	 * in VCL_Poll() which is called from the CLI thread.
@@ -298,21 +305,36 @@ static void
 vcl_set_state(struct vcl *vcl, const char *state)
 {
 	struct vrt_ctx ctx;
-	int warm;
 	unsigned hand = 0;
 
-	assert(state[0] == '0' || state[0] == '1');
-	warm = state[0] == '1' ? 1 : 0;
-	bprintf(vcl->state, "%s", state + 1);
-	if (warm == vcl->warm)
-		return;
-
-	vcl->warm = warm;
+	ASSERT_CLI();
+	AN(vcl->temp);
 
 	INIT_OBJ(&ctx, VRT_CTX_MAGIC);
 	ctx.handling = &hand;
-	(void)vcl->conf->event_vcl(&ctx,
-	    vcl->warm ? VCL_EVENT_WARM : VCL_EVENT_COLD);
+
+	switch(state[0]) {
+	case '0':
+		if (vcl->temp == vcl_temp_cold)
+			break;
+		if (vcl->busy == 0) {
+			vcl->temp = vcl_temp_cold;
+			(void)vcl->conf->event_vcl(&ctx, VCL_EVENT_COLD);
+		} else {
+			vcl->temp = vcl_temp_cooling;
+		}
+		break;
+	case '1':
+		if (vcl->temp != vcl_temp_cold)
+			vcl->temp = vcl_temp_warm;
+		else {
+			vcl->temp = vcl_temp_warm;
+			(void)vcl->conf->event_vcl(&ctx, VCL_EVENT_WARM);
+		}
+		break;
+	default:
+		WRONG("Wrong enum state");
+	}
 }
 
 static int
@@ -346,6 +368,8 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 	vcl->loaded_name = strdup(name);
 	XXXAN(vcl->loaded_name);
 
+	vcl->temp = vcl_temp_cold;
+
 	INIT_OBJ(&ctx, VRT_CTX_MAGIC);
 	ctx.method = VCL_MET_INIT;
 	ctx.handling = &hand;
@@ -366,6 +390,7 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 	}
 	VSB_delete(vsb);
 	vcl_set_state(vcl, state);
+	bprintf(vcl->state, "%s", state + 1);
 	assert(hand == VCL_RET_OK);
 	VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name);
 	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
@@ -413,9 +438,12 @@ VCL_Poll(void)
 	struct vcl *vcl, *vcl2;
 
 	ASSERT_CLI();
-	VTAILQ_FOREACH_SAFE(vcl, &vcl_head, list, vcl2)
+	VTAILQ_FOREACH_SAFE(vcl, &vcl_head, list, vcl2) {
+		if (vcl->temp == vcl_temp_cooling)
+			vcl_set_state(vcl, "0");
 		if (vcl->discard && vcl->busy == 0)
 			VCL_Nuke(vcl);
+	}
 }
 
 /*--------------------------------------------------------------------*/
@@ -437,9 +465,7 @@ ccf_config_list(struct cli *cli, const char * const *av, void *priv)
 		} else
 			flg = "available";
 		VCLI_Out(cli, "%-10s %4s/%s  %6u %s\n",
-		    flg,
-		    vcl->state, vcl->warm ? "warm" : "cold",
-		    vcl->busy, vcl->loaded_name);
+		    flg, vcl->state, vcl->temp, vcl->busy, vcl->loaded_name);
 	}
 }
 
@@ -467,6 +493,7 @@ ccf_config_state(struct cli *cli, const char * const *av, void *priv)
 	vcl = vcl_find(av[2]);
 	AN(vcl);			// MGT ensures this
 	vcl_set_state(vcl, av[3]);
+	bprintf(vcl->state, "%s", av[3] + 1);
 }
 
 static void __match_proto__(cli_func_t)
@@ -502,8 +529,8 @@ ccf_config_use(struct cli *cli, const char * const *av, void *priv)
 	ASSERT_CLI();
 	AZ(priv);
 	vcl = vcl_find(av[2]);
-	AN(vcl);			// MGT ensures this
-	AN(vcl->warm);			// MGT ensures this
+	AN(vcl);				// MGT ensures this
+	assert(vcl->temp == vcl_temp_warm);	// MGT ensures this
 	INIT_OBJ(&ctx, VRT_CTX_MAGIC);
 	ctx.handling = &hand;
 	vsb = VSB_new_auto();



From phk at FreeBSD.org  Mon Jun 22 10:55:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Jun 2015 12:55:04 +0200
Subject: [master] 0a14644 Hang backends from a list off the VCL that created
 them.
Message-ID: <E1Z6zNY-0000be-8r@project.varnish-software.com>


commit 0a14644f9709af2153db0e841dab908d0c81ea7c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 22 10:51:54 2015 +0000

    Hang backends from a list off the VCL that created them.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2b025ef..ab3cd4d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -102,6 +102,7 @@ enum {
 
 struct VSC_C_lck;
 struct ban;
+struct backend;
 struct busyobj;
 struct cli;
 struct cli_proto;
@@ -1077,6 +1078,8 @@ void VCL_Poll(void);
 void VCL_Ref(struct vcl *);
 void VCL_Refresh(struct vcl **);
 void VCL_Rel(struct vcl **);
+void VCL_AddBackend(struct vcl *, struct backend *);
+void VCL_DelBackend(struct vcl *, const struct backend *);
 const char *VCL_Return_Name(unsigned);
 
 #define VCL_MET_MAC(l,u,b) \
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index c3b9152..697b77a 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -59,6 +59,7 @@ struct backend {
 #define BACKEND_MAGIC		0x64c4c7c6
 
 	VTAILQ_ENTRY(backend)	list;
+	VTAILQ_ENTRY(backend)	vcl_list;
 	int			refcount;
 	struct lock		mtx;
 
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index e2350e2..3e07591 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -105,6 +105,8 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 	if (vrt->probe != NULL)
 		VBP_Insert(b, vrt->probe, vrt->hosthdr);
 
+	VCL_AddBackend(ctx->vcl, b);
+
 	return (b->director);
 }
 
@@ -149,13 +151,13 @@ VRT_delete_backend(VRT_CTX, struct director **dp)
 	ASSERT_CLI();
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(dp);
-	AN(*dp);
-
 	d = *dp;
 	*dp = NULL;
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC);
 
+	VCL_DelBackend(ctx->vcl, be);
+
 	if (be->probe != NULL)
 		VBP_Remove(be);
 
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index a6d7e80..865daac 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -39,6 +39,8 @@
 
 #include "cache.h"
 
+#include "cache_director.h"
+#include "cache_backend.h"
 #include "vcl.h"
 #include "vrt.h"
 #include "vcli.h"
@@ -59,6 +61,7 @@ struct vcl {
 	unsigned		busy;
 	unsigned		discard;
 	const char		*temp;
+	VTAILQ_HEAD(,backend)	backend_list;
 };
 
 /*
@@ -182,6 +185,29 @@ VCL_Rel(struct vcl **vcc)
 
 /*--------------------------------------------------------------------*/
 
+void
+VCL_AddBackend(struct vcl *vcl, struct backend *be)
+{
+
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
+	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
+	Lck_Lock(&vcl_mtx);
+	VTAILQ_INSERT_TAIL(&vcl->backend_list, be, vcl_list);
+	Lck_Unlock(&vcl_mtx);
+}
+
+void
+VCL_DelBackend(struct vcl *vcl, const struct backend *be)
+{
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
+	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
+	Lck_Lock(&vcl_mtx);
+	VTAILQ_REMOVE(&vcl->backend_list, be, vcl_list);
+	Lck_Unlock(&vcl_mtx);
+}
+
+/*--------------------------------------------------------------------*/
+
 static struct vcl *
 VCL_Open(const char *fn, struct vsb *msg)
 {
@@ -367,6 +393,7 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 
 	vcl->loaded_name = strdup(name);
 	XXXAN(vcl->loaded_name);
+	VTAILQ_INIT(&vcl->backend_list);
 
 	vcl->temp = vcl_temp_cold;
 



From phk at FreeBSD.org  Mon Jun 22 13:02:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Jun 2015 15:02:12 +0200
Subject: [master] 1fe06ba Make it cache_vcl's responsibility to send events to
 backends and to discard them with the VCL.
Message-ID: <E1Z71Ma-00028w-6x@project.varnish-software.com>


commit 1fe06ba636637bbb46617d339d83d3b5c0a4a2cd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 22 13:01:43 2015 +0000

    Make it cache_vcl's responsibility to send events to backends and to
    discard them with the VCL.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ab3cd4d..da22bf4 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1079,7 +1079,7 @@ void VCL_Ref(struct vcl *);
 void VCL_Refresh(struct vcl **);
 void VCL_Rel(struct vcl **);
 void VCL_AddBackend(struct vcl *, struct backend *);
-void VCL_DelBackend(struct vcl *, const struct backend *);
+void VCL_DelBackend(struct vcl *, struct backend *);
 const char *VCL_Return_Name(unsigned);
 
 #define VCL_MET_MAC(l,u,b) \
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 697b77a..b6b96e4 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -113,6 +113,10 @@ void VBE_fill_director(struct backend *be, const struct vrt_backend *vrt);
 
 /* cache_backend_cfg.c */
 unsigned VBE_Healthy(const struct backend *b, double *changed);
+#ifdef VCL_MET_MAX
+void VBE_Event(struct backend *, enum vcl_event_e);
+#endif
+void VBE_Delete(struct backend *be);
 
 /* cache_backend_poll.c */
 void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p,
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 3e07591..15332a1 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -38,19 +38,20 @@
 
 #include "cache.h"
 
+#include "vcl.h"
+#include "vrt.h"
+
 #include "cache_director.h"
 #include "cache_backend.h"
 #include "vcli.h"
 #include "vcli_priv.h"
 #include "vsa.h"
-#include "vcl.h"
-#include "vrt.h"
 #include "vtim.h"
 
 static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends);
 static struct lock backends_mtx;
 /*--------------------------------------------------------------------
- * Create a new director::backend instance.
+ * Create/Delete a new director::backend instance.
  */
 
 struct director *
@@ -111,19 +112,31 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 }
 
 void
-VRT_event_vbe(VRT_CTX, enum vcl_event_e ev, const struct director *d,
-    const struct vrt_backend *vrt)
+VRT_delete_backend(VRT_CTX, struct director **dp)
 {
+	struct director *d;
 	struct backend *be;
 
-	ASSERT_CLI();
-	(void)ev;
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(dp);
+	d = *dp;
+	*dp = NULL;
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
-	assert(d->priv2 == vrt);
-
 	CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC);
+	VCL_DelBackend(ctx->vcl, be);
+}
+
+/*---------------------------------------------------------------------
+ * These are for cross-calls with cache_vcl.c only.
+ */
+
+void
+VBE_Event(struct backend *be, enum vcl_event_e ev)
+{
+
+	ASSERT_CLI();
+	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
+
 	if (ev == VCL_EVENT_WARM) {
 		be->vsc = VSM_Alloc(sizeof *be->vsc,
 		    VSC_CLASS, VSC_type_vbe, be->display_name);
@@ -143,25 +156,14 @@ VRT_event_vbe(VRT_CTX, enum vcl_event_e ev, const struct director *d,
 }
 
 void
-VRT_delete_backend(VRT_CTX, struct director **dp)
+VBE_Delete(struct backend *be)
 {
-	struct director *d;
-	struct backend *be;
-
-	ASSERT_CLI();
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	AN(dp);
-	d = *dp;
-	*dp = NULL;
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC);
-
-	VCL_DelBackend(ctx->vcl, be);
+	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
 
 	if (be->probe != NULL)
 		VBP_Remove(be);
 
-	free(d->vcl_name);
+	free(be->director->vcl_name);
 
 	Lck_Lock(&backends_mtx);
 	VTAILQ_REMOVE(&backends, be, list);
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 865daac..02404fd 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -39,15 +39,16 @@
 
 #include "cache.h"
 
-#include "cache_director.h"
-#include "cache_backend.h"
 #include "vcl.h"
 #include "vrt.h"
+
+#include "cache_director.h"
+#include "cache_backend.h"
 #include "vcli.h"
 #include "vcli_priv.h"
 
 static const char * const vcl_temp_cold = "cold";
-static const char * const vcl_temp_warm = "warn";
+static const char * const vcl_temp_warm = "warm";
 static const char * const vcl_temp_cooling = "cooling";
 
 struct vcl {
@@ -197,13 +198,42 @@ VCL_AddBackend(struct vcl *vcl, struct backend *be)
 }
 
 void
-VCL_DelBackend(struct vcl *vcl, const struct backend *be)
+VCL_DelBackend(struct vcl *vcl, struct backend *be)
 {
 	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
 	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
 	Lck_Lock(&vcl_mtx);
 	VTAILQ_REMOVE(&vcl->backend_list, be, vcl_list);
 	Lck_Unlock(&vcl_mtx);
+	VBE_Delete(be);
+}
+
+static void
+vcl_BackendEvent(const struct vcl *vcl, enum vcl_event_e e)
+{
+	struct backend *be;
+
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
+	AZ(vcl->busy);
+
+	VTAILQ_FOREACH(be, &vcl->backend_list, vcl_list)
+		VBE_Event(be, e);
+}
+
+static void
+vcl_KillBackends(struct vcl *vcl)
+{
+	struct backend *be;
+
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
+	AZ(vcl->busy);
+	while (1) {
+		be = VTAILQ_FIRST(&vcl->backend_list);
+		if (be == NULL)
+			break;
+		VTAILQ_REMOVE(&vcl->backend_list, be, vcl_list);
+		VBE_Delete(be);
+	}
 }
 
 /*--------------------------------------------------------------------*/
@@ -346,6 +376,7 @@ vcl_set_state(struct vcl *vcl, const char *state)
 		if (vcl->busy == 0) {
 			vcl->temp = vcl_temp_cold;
 			(void)vcl->conf->event_vcl(&ctx, VCL_EVENT_COLD);
+			vcl_BackendEvent(vcl, VCL_EVENT_COLD);
 		} else {
 			vcl->temp = vcl_temp_cooling;
 		}
@@ -356,6 +387,7 @@ vcl_set_state(struct vcl *vcl, const char *state)
 		else {
 			vcl->temp = vcl_temp_warm;
 			(void)vcl->conf->event_vcl(&ctx, VCL_EVENT_WARM);
+			vcl_BackendEvent(vcl, VCL_EVENT_WARM);
 		}
 		break;
 	default:
@@ -411,6 +443,7 @@ VCL_Load(struct cli *cli, const char *name, const char *fn, const char *state)
 		if (VSB_len(vsb))
 			VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(vsb));
 		AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
+		vcl_KillBackends(vcl);
 		VCL_Close(&vcl);
 		VSB_delete(vsb);
 		return (1);
@@ -451,6 +484,7 @@ VCL_Nuke(struct vcl *vcl)
 	ctx.handling = &hand;
 	ctx.vcl = vcl;
 	AZ(vcl->conf->event_vcl(&ctx, VCL_EVENT_DISCARD));
+	vcl_KillBackends(vcl);
 	free(vcl->loaded_name);
 	VCL_Close(&vcl);
 	VSC_C_main->n_vcl--;
diff --git a/include/vrt.h b/include/vrt.h
index 27e738e..1dd5d0c 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -243,10 +243,6 @@ void VRT_synth_page(VRT_CTX, const char *, ...);
 
 /* Backend related */
 struct director *VRT_new_backend(VRT_CTX, const struct vrt_backend *);
-#ifdef VCL_RET_MAX
-void VRT_event_vbe(VRT_CTX, enum vcl_event_e, const struct director *,
-    const struct vrt_backend *);
-#endif
 void VRT_delete_backend(VRT_CTX, struct director **);
 
 /* Suckaddr related */
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 2bb3693..9ef1c95 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -412,11 +412,6 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 	VSB_printf(ifp->ini,
 	    "\t%s =\n\t    VRT_new_backend(ctx, &vgc_dir_priv_%s);",
 	    vgcname, vgcname);
-	VSB_printf(ifp->fin,
-	    "\t\tVRT_delete_backend(ctx, &%s);\n", vgcname);
-	VSB_printf(ifp->event,
-	    "\tVRT_event_vbe(ctx, ev, %s, &vgc_dir_priv_%s);",
-	    vgcname, vgcname);
 }
 
 /*--------------------------------------------------------------------



From phk at FreeBSD.org  Mon Jun 22 21:31:30 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Jun 2015 23:31:30 +0200
Subject: [master] 2e247f5 Rearrange #includes to get vrt.h before
 cache_backend.h
Message-ID: <E1Z79JS-00066Y-Fk@project.varnish-software.com>


commit 2e247f5248e24da00e2f983135dbd67a5098bfa3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 22 20:22:16 2015 +0000

    Rearrange #includes to get vrt.h before cache_backend.h

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index a057a14..789655f 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -36,11 +36,12 @@
 
 #include "cache.h"
 
-#include "cache_director.h"
-#include "cache_backend.h"
 #include "vrt.h"
 #include "vtcp.h"
 
+#include "cache_director.h"
+#include "cache_backend.h"
+
 #define FIND_TMO(tmx, dst, bo, be)					\
 	do {								\
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);			\
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 15332a1..b3766cd 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -39,15 +39,15 @@
 #include "cache.h"
 
 #include "vcl.h"
-#include "vrt.h"
-
-#include "cache_director.h"
-#include "cache_backend.h"
 #include "vcli.h"
 #include "vcli_priv.h"
+#include "vrt.h"
 #include "vsa.h"
 #include "vtim.h"
 
+#include "cache_director.h"
+#include "cache_backend.h"
+
 static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends);
 static struct lock backends_mtx;
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index fe6d735..d88f6e8 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -41,17 +41,17 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "binary_heap.h"
-
 #include "cache.h"
 
-#include "cache_director.h"
-#include "cache_backend.h"
+#include "binary_heap.h"
 #include "vcli_priv.h"
 #include "vrt.h"
-#include "vtim.h"
-#include "vtcp.h"
 #include "vsa.h"
+#include "vtcp.h"
+#include "vtim.h"
+
+#include "cache_director.h"
+#include "cache_backend.h"
 
 /* Default averaging rate, we want something pretty responsive */
 #define AVG_RATE			4
diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index c830fa5..c9d57d1 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -38,12 +38,14 @@
 
 #include "cache.h"
 
+#include "vrt.h"
+#include "vsa.h"
+#include "vtcp.h"
+#include "vtim.h"
+
 #include "cache_director.h"
 #include "cache_backend.h"
 #include "cache_pool.h"
-#include "vtcp.h"
-#include "vsa.h"
-#include "vtim.h"
 
 struct tcp_pool {
 	unsigned		magic;
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index b959ff6..8412d30 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -40,9 +40,11 @@
 #include <signal.h>
 
 #include "cache.h"
+
 #include "cache_filter.h"
 #include "common/heritage.h"
 
+#include "vrt.h"
 #include "cache_director.h"
 #include "cache_backend.h"
 #include "storage/storage.h"
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 1eae475..106fd03 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -35,13 +35,15 @@
 
 #include "cache/cache.h"
 
+#include "vcli_priv.h"
+#include "vrt.h"
+#include "vtcp.h"
+#include "vtim.h"
+
 #include "hash/hash_slinger.h"
 
 #include "cache/cache_director.h"
 #include "cache/cache_backend.h"
-#include "vcli_priv.h"
-#include "vtcp.h"
-#include "vtim.h"
 
 /*--------------------------------------------------------------------
  * Pass the request body to the backend
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index bf7ab67..45b137d 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -36,11 +36,13 @@
 
 #include "cache/cache.h"
 
-#include "cache/cache_director.h"
-#include "cache/cache_backend.h"
+#include "vrt.h"
 #include "vtcp.h"
 #include "vtim.h"
 
+#include "cache/cache_director.h"
+#include "cache/cache_backend.h"
+
 static struct lock pipestat_mtx;
 
 struct acct_pipe {



From phk at FreeBSD.org  Mon Jun 22 21:31:30 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Jun 2015 23:31:30 +0200
Subject: [master] 2afc943 Don't rely on continued access to the vrt_backend
 struct used to create a backend, that may be true of VCC compiled backends,
 but won't be for VMOD dynamic backends.
Message-ID: <E1Z79JS-00066f-H8@project.varnish-software.com>


commit 2afc943ab10511b5dab5d2947fd5a5db6545b067
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 22 21:29:55 2015 +0000

    Don't rely on continued access to the vrt_backend struct used to
    create a backend, that may be true of VCC compiled backends, but
    won't be for VMOD dynamic backends.
    
    Eliminate some excessive copying at the same time.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index da22bf4..706c92b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -119,7 +119,6 @@ struct req;
 struct sess;
 struct suckaddr;
 struct vbc;
-struct vrt_backend;
 struct vrt_priv;
 struct vsb;
 struct waitinglist;
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 789655f..5824504 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -83,7 +83,6 @@ vbe_dir_getfd(struct worker *wrk, const struct director *d, struct busyobj *bo)
 	struct vbc *vc;
 	struct backend *bp;
 	double tmod;
-	const struct vrt_backend *vrt;
 	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
 	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
 
@@ -92,7 +91,6 @@ vbe_dir_getfd(struct worker *wrk, const struct director *d, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 	AN(bp->vsc);
-	CAST_OBJ_NOTNULL(vrt, d->priv2, VRT_BACKEND_MAGIC);
 
 	if (!VBE_Healthy(bp, NULL)) {
 		// XXX: per backend stats ?
@@ -100,8 +98,7 @@ vbe_dir_getfd(struct worker *wrk, const struct director *d, struct busyobj *bo)
 		return (-1);
 	}
 
-	if (vrt->max_connections > 0 &&
-	    bp->n_conn >= vrt->max_connections) {
+	if (bp->max_connections > 0 && bp->n_conn >= bp->max_connections) {
 		// XXX: per backend stats ?
 		VSC_C_main->backend_busy++;
 		return (-1);
@@ -113,7 +110,7 @@ vbe_dir_getfd(struct worker *wrk, const struct director *d, struct busyobj *bo)
 		/* XXX: counter ? */
 		return (-1);
 
-	FIND_TMO(connect_timeout, tmod, bo, vrt);
+	FIND_TMO(connect_timeout, tmod, bo, bp);
 	vc = VBT_Get(bp->tcp_pool, tmod, bp, wrk);
 	if (vc == NULL) {
 		// XXX: Per backend stats ?
@@ -142,9 +139,9 @@ vbe_dir_getfd(struct worker *wrk, const struct director *d, struct busyobj *bo)
 	bo->htc->vbc = vc;
 	bo->htc->fd = vc->fd;
 	FIND_TMO(first_byte_timeout,
-	    bo->htc->first_byte_timeout, bo, vrt);
+	    bo->htc->first_byte_timeout, bo, bp);
 	FIND_TMO(between_bytes_timeout,
-	    bo->htc->between_bytes_timeout, bo, vrt);
+	    bo->htc->between_bytes_timeout, bo, bp);
 	return (vc->fd);
 }
 
@@ -200,12 +197,12 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
     struct busyobj *bo)
 {
 	int i, extrachance = 0;
-	const struct vrt_backend *vrt;
+	struct backend *bp;
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	CAST_OBJ_NOTNULL(vrt, d->priv2, VRT_BACKEND_MAGIC);
+	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
 	i = vbe_dir_getfd(wrk, d, bo);
 	if (i < 0) {
@@ -216,7 +213,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 	if (bo->htc->vbc->state == VBC_STATE_STOLEN)
 		extrachance = 1;
 
-	i = V1F_fetch_hdr(wrk, bo, vrt->hosthdr);
+	i = V1F_fetch_hdr(wrk, bo, bp->hosthdr);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
 	 * that the backend closed it before we get a request to it.
@@ -234,7 +231,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 			return (-1);
 		}
 		AN(bo->htc);
-		i = V1F_fetch_hdr(wrk, bo, vrt->hosthdr);
+		i = V1F_fetch_hdr(wrk, bo, bp->hosthdr);
 	}
 	if (i != 0) {
 		vbe_dir_finish(d, wrk, bo);
@@ -299,19 +296,17 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 /*--------------------------------------------------------------------*/
 
 void
-VBE_fill_director(struct backend *be, const struct vrt_backend *vrt)
+VBE_fill_director(struct backend *be)
 {
 	struct director *d;
 
 	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
-	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
 
 	INIT_OBJ(be->director, DIRECTOR_MAGIC);
 	d = be->director;
 	d->priv = be;
-	d->priv2 = vrt;
 	d->name = "backend";
-	REPLACE(d->vcl_name, vrt->vcl_name);
+	d->vcl_name = be->vcl_name;
 	d->http1pipe = vbe_dir_http1pipe;
 	d->healthy = vbe_dir_healthy;
 	d->gethdrs = vbe_dir_gethdrs;
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index b6b96e4..22655f9 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -63,15 +63,10 @@ struct backend {
 	int			refcount;
 	struct lock		mtx;
 
+	VRT_BACKEND_FIELDS()
+
 	const struct vcl	*vcl;
-	const char		*vcl_name;
 	char			*display_name;
-	const char		*ipv4_addr;
-	const char		*ipv6_addr;
-	const char		*port;
-
-	struct suckaddr		*ipv4;
-	struct suckaddr		*ipv6;
 
 	unsigned		n_conn;
 
@@ -109,7 +104,7 @@ struct vbc {
 };
 
 /* cache_backend.c */
-void VBE_fill_director(struct backend *be, const struct vrt_backend *vrt);
+void VBE_fill_director(struct backend *be);
 
 /* cache_backend_cfg.c */
 unsigned VBE_Healthy(const struct backend *b, double *changed);
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index b3766cd..d9cf6fe 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -42,7 +42,6 @@
 #include "vcli.h"
 #include "vcli_priv.h"
 #include "vrt.h"
-#include "vsa.h"
 #include "vtim.h"
 
 #include "cache_director.h"
@@ -64,6 +63,8 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
 
+	assert(vrt->ipv4_suckaddr != NULL || vrt->ipv6_suckaddr != NULL);
+
 	vcl = ctx->vcl;
 	AN(vcl);
 	AN(vrt->vcl_name);
@@ -74,24 +75,19 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 	XXXAN(b);
 	Lck_New(&b->mtx, lck_backend);
 
+#define DA(x)	do { if (vrt->x != NULL) REPLACE((b->x), (vrt->x)); } while (0)
+#define DN(x)	do { b->x = vrt->x; } while (0)
+	VRT_BACKEND_HANDLE();
+#undef DA
+#undef DN
+
 	bprintf(buf, "%s.%s", VCL_Name(vcl), vrt->vcl_name);
 	REPLACE(b->display_name, buf);
 
 	b->vcl = vcl;
-	b->vcl_name =  vrt->vcl_name;
-	b->ipv4_addr = vrt->ipv4_addr;
-	b->ipv6_addr = vrt->ipv6_addr;
-	b->port = vrt->port;
 
 	b->tcp_pool = VBT_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr);
 
-	if (vrt->ipv4_suckaddr != NULL)
-		b->ipv4 = VSA_Clone(vrt->ipv4_suckaddr);
-	if (vrt->ipv6_suckaddr != NULL)
-		b->ipv6 = VSA_Clone(vrt->ipv6_suckaddr);
-
-	assert(b->ipv4 != NULL || b->ipv6 != NULL);
-
 	b->healthy = 1;
 	b->health_changed = VTIM_real();
 	b->admin_health = ah_probe;
@@ -101,7 +97,7 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 	VSC_C_main->n_backend++;
 	Lck_Unlock(&backends_mtx);
 
-	VBE_fill_director(b, vrt);
+	VBE_fill_director(b);
 
 	if (vrt->probe != NULL)
 		VBP_Insert(b, vrt->probe, vrt->hosthdr);
@@ -170,8 +166,12 @@ VBE_Delete(struct backend *be)
 	VSC_C_main->n_backend--;
 	Lck_Unlock(&backends_mtx);
 
-	free(be->ipv4);
-	free(be->ipv6);
+#define DA(x)	do { if (be->x != NULL) free(be->x); } while (0)
+#define DN(x)	/**/
+	VRT_BACKEND_HANDLE();
+#undef DA
+#undef DN
+
 	free(be->display_name);
 	AZ(be->vsc);
 	VBT_Rel(&be->tcp_pool);
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index d88f6e8..d196eda 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -62,8 +62,6 @@ struct vbp_target {
 
 	struct backend			*backend;
 
-	struct tcp_pool			*tcp_pool;
-
 	struct vrt_backend_probe	probe;
 
 	char				*req;
@@ -217,7 +215,7 @@ vbp_poke(struct vbp_target *vt)
 	t_start = t_now = VTIM_real();
 	t_end = t_start + vt->probe.timeout;
 
-	s = VBT_Open(vt->tcp_pool, t_end - t_now, &sa);
+	s = VBT_Open(vt->backend->tcp_pool, t_end - t_now, &sa);
 	if (s < 0) {
 		/* Got no connection: failed */
 		return;
@@ -320,7 +318,6 @@ vbp_task(struct worker *wrk, void *priv)
 
 	Lck_Lock(&vbp_mtx);
 	if (vt->running < 0) {
-		VBT_Rel(&vt->tcp_pool);
 		free(vt->req);
 		FREE_OBJ(vt);
 	} else {
@@ -537,9 +534,6 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *p,
 	ALLOC_OBJ(vt, VBP_TARGET_MAGIC);
 	XXXAN(vt);
 
-	vt->tcp_pool = VBT_Ref(b->ipv4, b->ipv6);
-	AN(vt->tcp_pool);
-
 	vt->probe = *p;
 	vbp_set_defaults(vt);
 
@@ -572,7 +566,6 @@ VBP_Remove(struct backend *be)
 	}
 	Lck_Unlock(&vbp_mtx);
 	if (vt != NULL) {
-		VBT_Rel(&vt->tcp_pool);
 		free(vt->req);
 		FREE_OBJ(vt);
 	}
diff --git a/include/vrt.h b/include/vrt.h
index 1dd5d0c..2f0887d 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -166,29 +166,51 @@ struct vrt_backend_probe {
 	unsigned			initial;
 };
 
-/*
- * A backend is a host+port somewhere on the network
+/***********************************************************************
+ * We want the VCC to spit this structs out as const, but when VMODs
+ * come up with them we want to clone them into malloc'ed space which
+ * we can free again.
+ * We collect all the knowledge here by macroizing the fields and make
+ * a macro for handling them all.
+ * See also:  cache_backend.h & cache_backend_cfg.c
+ * One of those things...
  */
+
+#define VRT_BACKEND_FIELDS(rigid)				\
+	rigid char			*vcl_name;		\
+	rigid char			*ipv4_addr;		\
+	rigid char			*ipv6_addr;		\
+	rigid char			*port;			\
+	rigid char			*hosthdr;		\
+	double				connect_timeout;	\
+	double				first_byte_timeout;	\
+	double				between_bytes_timeout;	\
+	unsigned			max_connections;
+
+#define VRT_BACKEND_HANDLE()			\
+	do {					\
+		DA(vcl_name);			\
+		DA(ipv4_addr);			\
+		DA(ipv6_addr);			\
+		DA(port);			\
+		DA(hosthdr);			\
+		DN(connect_timeout);		\
+		DN(first_byte_timeout);		\
+		DN(between_bytes_timeout);	\
+		DN(max_connections);		\
+	} while(0)
+
 struct vrt_backend {
 	unsigned			magic;
 #define VRT_BACKEND_MAGIC		0x4799ce6b
-	const char			*vcl_name;
-	const char			*ipv4_addr;
-	const char			*ipv6_addr;
-	const char			*port;
-
+	VRT_BACKEND_FIELDS(const)
 	const struct suckaddr		*ipv4_suckaddr;
 	const struct suckaddr		*ipv6_suckaddr;
-
-	const char			*hosthdr;
-
-	double				connect_timeout;
-	double				first_byte_timeout;
-	double				between_bytes_timeout;
-	unsigned			max_connections;
 	const struct vrt_backend_probe	*probe;
 };
 
+/***********************************************************************/
+
 /*
  * other stuff.
  * XXX: document when bored



From phk at FreeBSD.org  Mon Jun 22 22:15:06 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 23 Jun 2015 00:15:06 +0200
Subject: [master] 4a7dc34 Give the vrt_backend_probe struct the same treatment
 as vrt_backend.
Message-ID: <E1Z79ze-000288-11@project.varnish-software.com>


commit 4a7dc346bdc0cfba8ad35d5084f517225b0185d0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 22 22:14:37 2015 +0000

    Give the vrt_backend_probe struct the same treatment as vrt_backend.

diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 22655f9..37ce9ce 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -115,7 +115,7 @@ void VBE_Delete(struct backend *be);
 
 /* cache_backend_poll.c */
 void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p,
-    const char *hosthdr);
+    struct tcp_pool *);
 void VBP_Remove(struct backend *b);
 void VBP_Control(const struct backend *b, int stop);
 void VBP_Status(struct cli *cli, const struct backend *, int details);
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index d9cf6fe..ebb4c42 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -100,7 +100,8 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 	VBE_fill_director(b);
 
 	if (vrt->probe != NULL)
-		VBP_Insert(b, vrt->probe, vrt->hosthdr);
+		VBP_Insert(b, vrt->probe,
+		    VBT_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr));
 
 	VCL_AddBackend(ctx->vcl, b);
 
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index d196eda..2cc9673 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -60,9 +60,10 @@ struct vbp_target {
 	unsigned			magic;
 #define VBP_TARGET_MAGIC		0x6b7cb656
 
-	struct backend			*backend;
+	VRT_BACKEND_PROBE_FIELDS()
 
-	struct vrt_backend_probe	probe;
+	struct backend			*backend;
+	struct tcp_pool			*tcp_pool;
 
 	char				*req;
 	int				req_len;
@@ -89,6 +90,20 @@ static struct lock			vbp_mtx;
 static pthread_cond_t			vbp_cond;
 static struct binheap			*vbp_heap;
 
+/*--------------------------------------------------------------------*/
+
+static void
+vbp_delete(struct vbp_target *vt)
+{
+#define DN(x)	/**/
+	VRT_BACKEND_PROBE_HANDLE();
+#undef DN
+	VBT_Rel(&vt->tcp_pool);
+	free(vt->req);
+	FREE_OBJ(vt);
+}
+
+
 /*--------------------------------------------------------------------
  * Record pokings...
  */
@@ -123,7 +138,7 @@ vbp_has_poked(struct vbp_target *vt)
 	}
 
 	u = vt->happy;
-	for (i = j = 0; i < vt->probe.window; i++) {
+	for (i = j = 0; i < vt->window; i++) {
 		if (u & 1)
 			j++;
 		u >>= 1;
@@ -149,7 +164,7 @@ vbp_update_backend(struct vbp_target *vt)
 #undef BITMAP
 		bits[i] = '\0';
 
-		if (vt->good >= vt->probe.threshold) {
+		if (vt->good >= vt->threshold) {
 			if (vt->backend->healthy)
 				logmsg = "Still healthy";
 			else {
@@ -167,7 +182,7 @@ vbp_update_backend(struct vbp_target *vt)
 		}
 		VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s",
 		    vt->backend->display_name, logmsg, bits,
-		    vt->good, vt->probe.threshold, vt->probe.window,
+		    vt->good, vt->threshold, vt->window,
 		    vt->last, vt->avg, vt->resp_buf);
 		if (vt->backend != NULL && vt->backend->vsc != NULL)
 			vt->backend->vsc->happy = vt->happy;
@@ -188,7 +203,7 @@ vbp_reset(struct vbp_target *vt)
 #include "tbl/backend_poll.h"
 #undef BITMAP
 
-	for (u = 0; u < vt->probe.initial; u++) {
+	for (u = 0; u < vt->initial; u++) {
 		vbp_start_poke(vt);
 		vt->happy |= 1;
 		vbp_has_poked(vt);
@@ -213,9 +228,9 @@ vbp_poke(struct vbp_target *vt)
 	const struct suckaddr *sa;
 
 	t_start = t_now = VTIM_real();
-	t_end = t_start + vt->probe.timeout;
+	t_end = t_start + vt->timeout;
 
-	s = VBT_Open(vt->backend->tcp_pool, t_end - t_now, &sa);
+	s = VBT_Open(vt->tcp_pool, t_end - t_now, &sa);
 	if (s < 0) {
 		/* Got no connection: failed */
 		return;
@@ -293,7 +308,7 @@ vbp_poke(struct vbp_target *vt)
 
 	i = sscanf(vt->resp_buf, "HTTP/%*f %u %s", &resp, buf);
 
-	if ((i == 1 || i == 2) && resp == vt->probe.exp_status)
+	if ((i == 1 || i == 2) && resp == vt->exp_status)
 		vt->happy |= 1;
 }
 
@@ -317,12 +332,10 @@ vbp_task(struct worker *wrk, void *priv)
 	vbp_update_backend(vt);
 
 	Lck_Lock(&vbp_mtx);
-	if (vt->running < 0) {
-		free(vt->req);
-		FREE_OBJ(vt);
-	} else {
+	if (vt->running < 0)
+		vbp_delete(vt);
+	else
 		vt->running = 0;
-	}
 	Lck_Unlock(&vbp_mtx);
 }
 
@@ -350,7 +363,7 @@ vbp_thread(struct worker *wrk, void *priv)
 			} else {
 				binheap_delete(vbp_heap, vt->heap_idx);
 				vt->running = 1;
-				vt->due = now + vt->probe.interval;
+				vt->due = now + vt->interval;
 				binheap_insert(vbp_heap, vt);
 				nxt = 0.0;
 				break;
@@ -401,7 +414,7 @@ vbp_health_one(struct cli *cli, const struct vbp_target *vt)
 
 	VCLI_Out(cli,
 	    "  Current states  good: %2u threshold: %2u window: %2u\n",
-	    vt->good, vt->probe.threshold, vt->probe.window);
+	    vt->good, vt->threshold, vt->window);
 	VCLI_Out(cli,
 	    "  Average response time of good probes: %.6f\n", vt->avg);
 	VCLI_Out(cli,
@@ -423,7 +436,7 @@ VBP_Status(struct cli *cli, const struct backend *be, int details)
 	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
 	vt = be->probe;
 	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-	VCLI_Out(cli, "%d/%d", vt->good, vt->probe.window);
+	VCLI_Out(cli, "%d/%d", vt->good, vt->window);
 	if (details) {
 		VCLI_Out(cli, "\n");
 		vbp_health_one(cli, vt);
@@ -435,20 +448,21 @@ VBP_Status(struct cli *cli, const struct backend *be, int details)
  */
 
 static void
-vbp_build_req(struct vbp_target *vt, const char *hosthdr)
+vbp_build_req(struct vbp_target *vt, const struct vrt_backend_probe *vbp,
+    const struct backend *be)
 {
 	struct vsb *vsb;
 
 	vsb = VSB_new_auto();
 	AN(vsb);
 	VSB_clear(vsb);
-	if(vt->probe.request != NULL) {
-		VSB_cat(vsb, vt->probe.request);
+	if(vbp->request != NULL) {
+		VSB_cat(vsb, vbp->request);
 	} else {
 		VSB_printf(vsb, "GET %s HTTP/1.1\r\n",
-		    vt->probe.url != NULL ?  vt->probe.url : "/");
-		if (hosthdr != NULL)
-			VSB_printf(vsb, "Host: %s\r\n", hosthdr);
+		    vbp->url != NULL ?  vbp->url : "/");
+		if (be->hosthdr != NULL)
+			VSB_printf(vsb, "Host: %s\r\n", be->hosthdr);
 		VSB_printf(vsb, "Connection: close\r\n");
 		VSB_printf(vsb, "\r\n");
 	}
@@ -465,25 +479,29 @@ vbp_build_req(struct vbp_target *vt, const char *hosthdr)
  */
 
 static void
-vbp_set_defaults(struct vbp_target *vt)
+vbp_set_defaults(struct vbp_target *vt, const struct vrt_backend_probe *vp)
 {
 
-	if (vt->probe.timeout == 0.0)
-		vt->probe.timeout = 2.0;
-	if (vt->probe.interval == 0.0)
-		vt->probe.interval = 5.0;
-	if (vt->probe.window == 0)
-		vt->probe.window = 8;
-	if (vt->probe.threshold == 0)
-		vt->probe.threshold = 3;
-	if (vt->probe.exp_status == 0)
-		vt->probe.exp_status = 200;
-
-	if (vt->probe.initial == ~0U)
-		vt->probe.initial = vt->probe.threshold - 1;
-
-	if (vt->probe.initial > vt->probe.threshold)
-		vt->probe.initial = vt->probe.threshold;
+#define DN(x)	do { vt->x = vp->x; } while (0)
+	VRT_BACKEND_PROBE_HANDLE();
+#undef DN
+
+	if (vt->timeout == 0.0)
+		vt->timeout = 2.0;
+	if (vt->interval == 0.0)
+		vt->interval = 5.0;
+	if (vt->window == 0)
+		vt->window = 8;
+	if (vt->threshold == 0)
+		vt->threshold = 3;
+	if (vt->exp_status == 0)
+		vt->exp_status = 200;
+
+	if (vt->initial == ~0U)
+		vt->initial = vt->threshold - 1;
+
+	if (vt->initial > vt->threshold)
+		vt->initial = vt->threshold;
 }
 
 /*--------------------------------------------------------------------
@@ -520,28 +538,27 @@ VBP_Control(const struct backend *be, int enable)
  */
 
 void
-VBP_Insert(struct backend *b, const struct vrt_backend_probe *p,
-    const char *hosthdr)
+VBP_Insert(struct backend *b, const struct vrt_backend_probe *vp,
+    struct tcp_pool *tp)
 {
 	struct vbp_target *vt;
 
 	ASSERT_CLI();
 	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-	CHECK_OBJ_NOTNULL(p, VRT_BACKEND_PROBE_MAGIC);
+	CHECK_OBJ_NOTNULL(vp, VRT_BACKEND_PROBE_MAGIC);
 
 	AZ(b->probe);
 
 	ALLOC_OBJ(vt, VBP_TARGET_MAGIC);
 	XXXAN(vt);
 
-	vt->probe = *p;
-	vbp_set_defaults(vt);
-
-	vbp_build_req(vt, hosthdr);
-
+	vt->tcp_pool = tp;
 	vt->backend = b;
 	b->probe = vt;
 
+	vbp_set_defaults(vt, vp);
+	vbp_build_req(vt, vp, b);
+
 	vbp_reset(vt);
 	vbp_update_backend(vt);
 }
@@ -565,10 +582,8 @@ VBP_Remove(struct backend *be)
 		vt = NULL;
 	}
 	Lck_Unlock(&vbp_mtx);
-	if (vt != NULL) {
-		free(vt->req);
-		FREE_OBJ(vt);
-	}
+	if (vt != NULL)
+		vbp_delete(vt);
 }
 
 /*-------------------------------------------------------------------*/
diff --git a/include/vrt.h b/include/vrt.h
index 2f0887d..63d1a85 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -146,26 +146,9 @@ struct gethdr_s {
 	const char	*what;
 };
 
-/*
- * A backend probe specification
- */
-
 extern const void * const vrt_magic_string_end;
 extern const void * const vrt_magic_string_unset;
 
-struct vrt_backend_probe {
-	unsigned			magic;
-#define VRT_BACKEND_PROBE_MAGIC		0x84998490
-	const char			*url;
-	const char			*request;
-	double				timeout;
-	double				interval;
-	unsigned			exp_status;
-	unsigned			window;
-	unsigned			threshold;
-	unsigned			initial;
-};
-
 /***********************************************************************
  * We want the VCC to spit this structs out as const, but when VMODs
  * come up with them we want to clone them into malloc'ed space which
@@ -209,6 +192,32 @@ struct vrt_backend {
 	const struct vrt_backend_probe	*probe;
 };
 
+#define VRT_BACKEND_PROBE_FIELDS(rigid)				\
+	double				timeout;		\
+	double				interval;		\
+	unsigned			exp_status;		\
+	unsigned			window;			\
+	unsigned			threshold;		\
+	unsigned			initial;
+
+#define VRT_BACKEND_PROBE_HANDLE()		\
+	do {					\
+		DN(timeout);			\
+		DN(interval);			\
+		DN(exp_status);			\
+		DN(window);			\
+		DN(threshold);			\
+		DN(initial);			\
+	} while (0)
+
+struct vrt_backend_probe {
+	unsigned			magic;
+#define VRT_BACKEND_PROBE_MAGIC		0x84998490
+	const char			*url;
+	const char			*request;
+	VRT_BACKEND_PROBE_FIELDS(const)
+};
+
 /***********************************************************************/
 
 /*



From phk at FreeBSD.org  Tue Jun 23 07:08:49 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 23 Jun 2015 09:08:49 +0200
Subject: [master] eb9ecf9 avoid a double free
Message-ID: <E1Z7IK9-0007Z0-6y@project.varnish-software.com>


commit eb9ecf9186a956d703bb90745e3df686fdbb90d7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 23 07:08:39 2015 +0000

    avoid a double free

diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index ebb4c42..e56ec77 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -160,8 +160,6 @@ VBE_Delete(struct backend *be)
 	if (be->probe != NULL)
 		VBP_Remove(be);
 
-	free(be->director->vcl_name);
-
 	Lck_Lock(&backends_mtx);
 	VTAILQ_REMOVE(&backends, be, list);
 	VSC_C_main->n_backend--;



From phk at FreeBSD.org  Tue Jun 23 09:06:25 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 23 Jun 2015 11:06:25 +0200
Subject: [master] 91dfd24 Add support for a default backend probe.
Message-ID: <E1Z7K9x-00085r-MH@project.varnish-software.com>


commit 91dfd24d30bc7ae9f67d33ffbfb5cb840f8bd492
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 23 09:05:32 2015 +0000

    Add support for a default backend probe.
    
    If you add a probe named "default" all backends will get it automatically,
    unless they have their own .probe configuration.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 706c92b..71ac981 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1069,6 +1069,7 @@ void VRY_Finish(struct req *req, enum vry_finish_flag);
 
 /* cache_vcl.c */
 struct director *VCL_DefaultDirector(const struct vcl *);
+const struct vrt_backend_probe *VCL_DefaultProbe(const struct vcl *);
 void VCL_Init(void);
 const char *VCL_Method_Name(unsigned);
 const char *VCL_Name(const struct vcl *);
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index e56ec77..7291cd4 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -59,6 +59,7 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 	struct backend *b;
 	char buf[128];
 	struct vcl *vcl;
+	const struct vrt_backend_probe *vbp;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
@@ -99,8 +100,11 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 
 	VBE_fill_director(b);
 
-	if (vrt->probe != NULL)
-		VBP_Insert(b, vrt->probe,
+	vbp = vrt->probe;
+	if (vbp == NULL)
+		vbp = VCL_DefaultProbe(vcl);
+	if (vbp != NULL)
+		VBP_Insert(b, vbp,
 		    VBT_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr));
 
 	VCL_AddBackend(ctx->vcl, b);
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 02404fd..40b630a 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -312,18 +312,27 @@ VCL_TestLoad(const char *fn)
 /*--------------------------------------------------------------------*/
 
 struct director *
-VCL_DefaultDirector(const struct vcl *vcc)
+VCL_DefaultDirector(const struct vcl *vcl)
 {
 
-	AN(vcc);
-	return (*vcc->conf->default_director);
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
+	return (*vcl->conf->default_director);
 }
 
 const char *
-VCL_Name(const struct vcl *vcc)
+VCL_Name(const struct vcl *vcl)
 {
-	AN(vcc);
-	return (vcc->loaded_name);
+
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
+	return (vcl->loaded_name);
+}
+
+const struct vrt_backend_probe *
+VCL_DefaultProbe(const struct vcl *vcl)
+{
+	
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
+	return (vcl->conf->default_probe);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishtest/tests/c00035.vtc b/bin/varnishtest/tests/c00035.vtc
index 5eab71d..4bffe36 100644
--- a/bin/varnishtest/tests/c00035.vtc
+++ b/bin/varnishtest/tests/c00035.vtc
@@ -6,26 +6,21 @@ server s1 -repeat 40 {
 } -start
 
 varnish v1 -vcl {
+	probe default {
+		.window = 8;
+		.initial = 7;
+		.threshold = 8;
+		.interval = 0.1s;
+	}
 	backend s1 {
 		.host = "${s1_addr}";
 		.port = "${s1_port}";
-		.probe = {
-			.window = 8;
-			.initial = 7;
-			.threshold = 8;
-			.interval = 0.1s;
-		}
 	}
 } -start
 
 delay 1
 
-varnish v1 -vcl {
-	backend s1 {
-		.host = "${s1_addr}";
-		.port = "${s1_port}";
-	}
-} -cliok "vcl.use vcl2" -cliok "vcl.discard vcl1"
+varnish v1 -vcl+backend { } -cliok "vcl.use vcl2" -cliok "vcl.discard vcl1"
 
 delay 1
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 345ae05..39810fc 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -1055,18 +1055,19 @@ fo.write("\n" + tbl40("#define VCL_RET_MAX", "%d\n" % n))
 
 fo.write("""
 struct VCL_conf {
-	unsigned	magic;
-#define VCL_CONF_MAGIC	0x7406c509	/* from /dev/random */
+	unsigned			magic;
+#define VCL_CONF_MAGIC			0x7406c509	/* from /dev/random */
 
-	struct director	**default_director;
-	unsigned	nref;
-	struct vrt_ref	*ref;
+	struct director			**default_director;
+	const struct vrt_backend_probe	*default_probe;
+	unsigned			nref;
+	struct vrt_ref			*ref;
 
-	unsigned	nsrc;
-	const char	**srcname;
-	const char	**srcbody;
+	unsigned			nsrc;
+	const char			**srcname;
+	const char			**srcbody;
 
-	vcl_event_f	*event_vcl;
+	vcl_event_f			*event_vcl;
 """)
 
 for i in returns:
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 9ef1c95..c0501a3 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -91,12 +91,14 @@ vcc_ProbeRedef(struct vcc *tl, struct token **t_did,
 }
 
 static void
-vcc_ParseProbeSpec(struct vcc *tl)
+vcc_ParseProbeSpec(struct vcc *tl, const struct token *nm, char **name)
 {
 	struct fld_spec *fs;
 	struct token *t_field;
 	struct token *t_did = NULL, *t_window = NULL, *t_threshold = NULL;
 	struct token *t_initial = NULL;
+	struct vsb *vsb;
+	char *retval;
 	unsigned window, threshold, initial, status;
 	double t;
 
@@ -113,12 +115,23 @@ vcc_ParseProbeSpec(struct vcc *tl)
 
 	SkipToken(tl, '{');
 
+	vsb = VSB_new_auto();
+	AN(vsb);
+	if (nm != NULL)
+		VSB_printf(vsb, "vgc_probe_%.*s", PF(nm));
+	else
+		VSB_printf(vsb, "vgc_probe__%d", tl->nprobe++);
+	AZ(VSB_finish(vsb));
+	retval = TlDup(tl, VSB_data(vsb));
+	VSB_delete(vsb);
+	if (name != NULL)
+		*name = retval;
+
 	window = 0;
 	threshold = 0;
 	initial = 0;
 	status = 0;
-	Fh(tl, 0, "static const struct vrt_backend_probe vgc_probe__%d = {\n",
-	    tl->nprobe++);
+	Fh(tl, 0, "static const struct vrt_backend_probe %s = {\n", retval);
 	Fh(tl, 0, "\t.magic = VRT_BACKEND_PROBE_MAGIC,\n");
 	while (tl->t->tok != '}') {
 
@@ -237,6 +250,7 @@ vcc_ParseProbe(struct vcc *tl)
 {
 	struct token *t_probe;
 	int i;
+	char *p;
 
 	vcc_NextToken(tl);		/* ID: probe */
 
@@ -250,9 +264,11 @@ vcc_ParseProbe(struct vcc *tl)
 		vcc_ErrWhere(tl, t_probe);
 	}
 
-	Fh(tl, 0, "\n#define vgc_probe_%.*s\tvgc_probe__%d\n",
-	    PF(t_probe), tl->nprobe);
-	vcc_ParseProbeSpec(tl);
+	vcc_ParseProbeSpec(tl, t_probe, &p);
+	if (vcc_IdIs(t_probe, "default")) {
+		vcc_AddRef(tl, t_probe, SYM_PROBE);
+		tl->default_probe = p;
+	}
 }
 
 /*--------------------------------------------------------------------
@@ -271,6 +287,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 	struct fld_spec *fs;
 	struct inifin *ifp;
 	struct vsb *vsb;
+	char *p;
 	unsigned u;
 	double t;
 
@@ -356,8 +373,8 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 			SkipToken(tl, ';');
 			Fb(tl, 0, "\t.max_connections = %u,\n", u);
 		} else if (vcc_IdIs(t_field, "probe") && tl->t->tok == '{') {
-			Fb(tl, 0, "\t.probe = &vgc_probe__%d,\n", tl->nprobe);
-			vcc_ParseProbeSpec(tl);
+			vcc_ParseProbeSpec(tl, NULL, &p);
+			Fb(tl, 0, "\t.probe = &%s,\n", p);
 			ERRCHK(tl);
 		} else if (vcc_IdIs(t_field, "probe") && tl->t->tok == ID) {
 			Fb(tl, 0, "\t.probe = &vgc_probe_%.*s,\n", PF(tl->t));
@@ -444,7 +461,7 @@ vcc_ParseBackend(struct vcc *tl)
 	vcc_NextToken(tl);
 
 	sprintf(vgcname, "vgc_backend_%.*s", PF(t_be));
-	Fh(tl, 0, "static struct director *%s;\n", vgcname);
+	Fh(tl, 0, "\nstatic struct director *%s;\n", vgcname);
 
 	sym = VCC_GetSymbolTok(tl, t_be, SYM_BACKEND);
 	AN(sym);
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 511f20e..36f813a 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -405,6 +405,8 @@ EmitStruct(const struct vcc *tl)
 	Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n");
 	Fc(tl, 0, "\t.event_vcl = VGC_Event,\n");
 	Fc(tl, 0, "\t.default_director = &%s,\n", tl->default_director);
+	if (tl->default_probe != NULL)
+		Fc(tl, 0, "\t.default_probe = &%s,\n", tl->default_probe);
 	Fc(tl, 0, "\t.ref = VGC_ref,\n");
 	Fc(tl, 0, "\t.nref = VGC_NREFS,\n");
 	Fc(tl, 0, "\t.nsrc = VGC_NSRCS,\n");
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index a222b9b..86f731d 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -198,6 +198,7 @@ struct vcc {
 
 	const char		*default_director;
 	struct token		*t_default_director;
+	const char		*default_probe;
 
 	unsigned		unique;
 



From phk at FreeBSD.org  Tue Jun 23 11:16:24 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 23 Jun 2015 13:16:24 +0200
Subject: [master] a4747b3 Add a parameter to tune the backend connection idle
 timeout.
Message-ID: <E1Z7MBk-0006Md-0S@project.varnish-software.com>


commit a4747b3e39a53bdc250f1a404c410ad534510fa3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 23 11:16:03 2015 +0000

    Add a parameter to tune the backend connection idle timeout.

diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index c9d57d1..75f216d 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -60,7 +60,6 @@ struct tcp_pool {
 	struct lock		mtx;
 
 	struct waitfor		waitfor;
-	volatile double		timeout;
 
 	VTAILQ_HEAD(, vbc)	connlist;
 	int			n_conn;
@@ -165,10 +164,9 @@ VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6)
 	VTAILQ_INIT(&tp->connlist);
 	VTAILQ_INIT(&tp->killlist);
 	VTAILQ_INSERT_HEAD(&pools, tp, list);
-	tp->timeout = 60;
 	INIT_OBJ(&tp->waitfor, WAITFOR_MAGIC);
 	tp->waitfor.func = tcp_handle;
-	tp->waitfor.tmo = &tp->timeout;
+	tp->waitfor.tmo = &cache_param->backend_idle_timeout;
 	return (tp);
 }
 
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 40b630a..6df2a2f 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -330,7 +330,7 @@ VCL_Name(const struct vcl *vcl)
 const struct vrt_backend_probe *
 VCL_DefaultProbe(const struct vcl *vcl)
 {
-	
+
 	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
 	return (vcl->conf->default_probe);
 }
diff --git a/include/tbl/params.h b/include/tbl/params.h
index 1581374..22addb0 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -195,6 +195,20 @@ PARAM(
 	/* func */	NULL
 )
 
+PARAM(
+	/* name */	backend_idle_timeout,
+	/* tweak */	timeout,
+	/* min */	"1",
+	/* max */	NULL,
+	/* default */	"60",
+	/* units */	"seconds",
+	/* flags */	0,
+	/* s-text */
+	"Timeout before we close unused backend connections.",
+	/* l-text */	"",
+	/* func */	NULL
+)
+
 /**********************************************************************/
 #if 0 /* NOT YET */
 



From fgsch at lodoss.net  Tue Jun 23 15:14:58 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 23 Jun 2015 17:14:58 +0200
Subject: [master] 606b2d6 VXID is the default varnishlog grouping mode
Message-ID: <E1Z7Puc-0002AF-Bw@project.varnish-software.com>


commit 606b2d6d3a7a011a147f8691a13bb3308dc8fc5d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Jun 23 16:13:50 2015 +0100

    VXID is the default varnishlog grouping mode
    
    Submitted by:       controversy187 via github

diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst
index ec73724..2c15453 100644
--- a/doc/sphinx/reference/vsl-query.rst
+++ b/doc/sphinx/reference/vsl-query.rst
@@ -60,14 +60,14 @@ The grouping modes are:
   Transactions are grouped by request, where the set will include the
   request itself as well as any backend requests or ESI-subrequests.
   Session data and non-transactional data (VXID == 0) is not
-  reported. This is the default.
+  reported.
 
 * VXID
 
   Transactions are not grouped, so each VXID is reported in it's
   entirety. Sessions, requests, ESI-requests and backend requests are
   all reported individually. Non-transactional data is not reported
-  (VXID == 0).
+  (VXID == 0). This is the default.
 
 * Raw
 



From phk at FreeBSD.org  Wed Jun 24 09:59:14 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 11:59:14 +0200
Subject: [master] bd70241 Make sure we don't finish off a backend connection
 while the waiter is still engaged with it.
Message-ID: <E1Z7hSc-0002Xo-MZ@project.varnish-software.com>


commit bd702410005f0963dab2ed93c1407ddaa577db12
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 09:56:30 2015 +0000

    Make sure we don't finish off a backend connection while the waiter
    is still engaged with it.
    
    Simplify the "do a single retry" logic slightly while here.
    
    Add more asserts.
    
    This probably fixes #1675.
    
    Initial diagnosis by:	martin

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 5824504..f85e4ca 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -169,8 +169,12 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
 	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
+	CHECK_OBJ_ORNULL(bo->htc->vbc, VBC_MAGIC);
 	if (bo->htc->vbc == NULL)
 		return;
+	if (bo->htc->vbc->state != VBC_STATE_USED)
+		VBT_Wait(wrk, bo->htc->vbc);
+	CHECK_OBJ_NOTNULL(bo->htc->vbc->backend, BACKEND_MAGIC);
 	bo->htc->vbc->backend = NULL;
 	if (bo->doclose != SC_NULL) {
 		VSLb(bo->vsl, SLT_BackendClose, "%d %s", bo->htc->vbc->fd,
@@ -196,7 +200,7 @@ static int __match_proto__(vdi_gethdrs_f)
 vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
     struct busyobj *bo)
 {
-	int i, extrachance = 0;
+	int i, extrachance = 1;
 	struct backend *bp;
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
@@ -204,43 +208,38 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
-	i = vbe_dir_getfd(wrk, d, bo);
-	if (i < 0) {
-		VSLb(bo->vsl, SLT_FetchError, "no backend connection");
-		return (-1);
-	}
-	AN(bo->htc);
-	if (bo->htc->vbc->state == VBC_STATE_STOLEN)
-		extrachance = 1;
-
-	i = V1F_fetch_hdr(wrk, bo, bp->hosthdr);
-	/*
-	 * If we recycle a backend connection, there is a finite chance
-	 * that the backend closed it before we get a request to it.
-	 * Do a single retry in that case.
-	 */
-	if (i == 1 && extrachance) {
-		vbe_dir_finish(d, wrk, bo);
-		AZ(bo->htc);
-		VSC_C_main->backend_retry++;
-		bo->doclose = SC_NULL;
+	do {
 		i = vbe_dir_getfd(wrk, d, bo);
 		if (i < 0) {
 			VSLb(bo->vsl, SLT_FetchError, "no backend connection");
-			bo->htc = NULL;
 			return (-1);
 		}
 		AN(bo->htc);
+		if (bo->htc->vbc->state != VBC_STATE_STOLEN)
+			extrachance = 0;
+
 		i = V1F_fetch_hdr(wrk, bo, bp->hosthdr);
-	}
-	if (i != 0) {
+		/*
+		 * If we recycled a backend connection, there is a finite chance
+		 * that the backend closed it before we got the bereq to it.
+		 * In that case do a single automatic retry if req.boy allows.
+		 */
+		if (i == 0) {
+			AN(bo->htc->vbc);
+			return (0);
+		}
 		vbe_dir_finish(d, wrk, bo);
-		bo->doclose = SC_NULL;
 		AZ(bo->htc);
-	} else {
-		AN(bo->htc->vbc);
-	}
-	return (i);
+		if (i < 0)
+			break;
+		if (bo->req != NULL &&
+		    bo->req->req_body_status != REQ_BODY_NONE &&
+		    bo->req->req_body_status != REQ_BODY_CACHED)
+			break;
+		VSC_C_main->backend_retry++;
+		bo->doclose = SC_NULL;
+	} while (extrachance);
+	return (-1);
 }
 
 static int __match_proto__(vdi_getbody_f)
diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index 75f216d..9a658dd 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -401,11 +401,13 @@ VBT_Wait(struct worker *wrk, const struct vbc *vbc)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(vbc->backend, BACKEND_MAGIC);
 	tp = vbc->tcp_pool;
 	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
 	assert(vbc->wrk == wrk);
 	Lck_Lock(&tp->mtx);
 	while (vbc->state == VBC_STATE_STOLEN)
 		AZ(Lck_CondWait(&wrk->cond, &tp->mtx, 0));
+	assert(vbc->state == VBC_STATE_USED);
 	Lck_Unlock(&tp->mtx);
 }
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 106fd03..12f18f1 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -80,7 +80,6 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
 {
 	struct http *hp;
 	enum htc_status_e hs;
-	int retry = 1;
 	int j, first;
 	ssize_t i;
 	struct http_conn *htc;
@@ -120,14 +119,12 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
 			V1L_Chunked(wrk);
 		i = VRB_Iterate(bo->req, vbf_iter_req_body, bo);
 
-		if (bo->req->req_body_status == REQ_BODY_TAKEN) {
-			retry = -1;
-		} else if (bo->req->req_body_status == REQ_BODY_FAIL) {
+		if (bo->req->req_body_status == REQ_BODY_FAIL) {
+			assert(i < 0);
 			VSLb(bo->vsl, SLT_FetchError,
 			    "req.body read error: %d (%s)",
 			    errno, strerror(errno));
 			bo->req->doclose = SC_RX_BODY;
-			retry = -1;
 		}
 		if (do_chunked)
 			V1L_EndChunk(wrk);
@@ -139,7 +136,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
 		    errno, strerror(errno));
 		VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
 		bo->doclose = SC_TX_ERROR;
-		return (retry);
+		return (1);
 	}
 	VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
 
@@ -181,10 +178,9 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
 			VSLb(bo->vsl, SLT_FetchError, "http %sread error: EOF",
 			    first ? "first " : "");
 			bo->doclose = SC_RX_TIMEOUT;
-			return (retry);
+			return (first ? 1 : -1);
 		}
 		if (first) {
-			retry = -1;
 			first = 0;
 			VTCP_set_read_timeout(htc->fd,
 			    htc->between_bytes_timeout);



From phk at FreeBSD.org  Wed Jun 24 11:41:10 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 13:41:10 +0200
Subject: [master] 746b4da Move bo->doclose to bo->htc->doclose which is more
 correct.
Message-ID: <E1Z7j3G-0002dD-Hm@project.varnish-software.com>


commit 746b4da3cafc6146fbcea65dbc741ccf7353d66e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 11:08:36 2015 +0000

    Move bo->doclose to bo->htc->doclose which is more correct.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 71ac981..8883de1 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -225,6 +225,7 @@ struct http_conn {
 #define HTTP_CONN_MAGIC		0x3e19edd1
 
 	int			fd;
+	enum sess_close		doclose;
 	unsigned		maxbytes;
 	unsigned		maxhdr;
 	struct ws		*ws;
@@ -481,8 +482,6 @@ struct busyobj {
 
 	struct pool_task	fetch_task;
 
-	enum sess_close		doclose;
-
 #define BO_FLAG(l, r, w, d) unsigned	l:1;
 #include "tbl/bo_flags.h"
 #undef BO_FLAG
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index f85e4ca..66c79d8 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -109,6 +109,7 @@ vbe_dir_getfd(struct worker *wrk, const struct director *d, struct busyobj *bo)
 	if (bo->htc == NULL)
 		/* XXX: counter ? */
 		return (-1);
+	bo->htc->doclose = SC_NULL;
 
 	FIND_TMO(connect_timeout, tmod, bo, bp);
 	vc = VBT_Get(bp->tcp_pool, tmod, bp, wrk);
@@ -176,7 +177,7 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 		VBT_Wait(wrk, bo->htc->vbc);
 	CHECK_OBJ_NOTNULL(bo->htc->vbc->backend, BACKEND_MAGIC);
 	bo->htc->vbc->backend = NULL;
-	if (bo->doclose != SC_NULL) {
+	if (bo->htc->doclose != SC_NULL) {
 		VSLb(bo->vsl, SLT_BackendClose, "%d %s", bo->htc->vbc->fd,
 		    bp->display_name);
 		VBT_Close(bp->tcp_pool, &bo->htc->vbc);
@@ -237,7 +238,6 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 		    bo->req->req_body_status != REQ_BODY_CACHED)
 			break;
 		VSC_C_main->backend_retry++;
-		bo->doclose = SC_NULL;
 	} while (extrachance);
 	return (-1);
 }
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index d899378..5dc4abe 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -143,7 +143,6 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req)
 	VCL_Ref(bo->vcl);
 
 	bo->t_first = bo->t_prev = NAN;
-	bo->doclose = SC_NULL;
 
 	memcpy(bo->digest, req->digest, sizeof bo->digest);
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3eed33b..ce83b99 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -169,7 +169,6 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
 
 	assert(bo->state == BOS_INVALID);
-	assert(bo->doclose == SC_NULL);
 	AZ(bo->storage_hint);
 
 	HTTP_Setup(bo->bereq0, bo->ws, bo->vsl, SLT_BereqMethod);
@@ -228,7 +227,6 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 
 	/* VDI_Finish must have been called before */
 	assert(bo->director_state == DIR_S_NULL);
-	bo->doclose = SC_NULL;
 
 	/* reset other bo attributes - See VBO_GetBusyObj */
 	bo->storage_hint = NULL;
@@ -261,7 +259,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	assert(bo->doclose == SC_NULL);
 	AZ(bo->storage_hint);
 
 	if (bo->do_pass)
@@ -372,7 +369,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	}
 
 	if (bo->htc->body_status == BS_ERROR) {
-		bo->doclose = SC_RX_BODY;
+		bo->htc->doclose = SC_RX_BODY;
 		VDI_Finish(bo->wrk, bo);
 		VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
 		assert(bo->director_state == DIR_S_NULL);
@@ -414,7 +411,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 			 */
 			VSLb(bo->vsl, SLT_Error,
 			    "304 response but not conditional fetch");
-			bo->doclose = SC_RX_BAD;
+			bo->htc->doclose = SC_RX_BAD;
 			VDI_Finish(bo->wrk, bo);
 			return (F_STP_FAIL);
 		}
@@ -429,14 +426,14 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
 
 	if (wrk->handling == VCL_RET_ABANDON) {
-		bo->doclose = SC_RESP_CLOSE;
+		bo->htc->doclose = SC_RESP_CLOSE;
 		VDI_Finish(bo->wrk, bo);
 		return (F_STP_FAIL);
 	}
 
 	if (wrk->handling == VCL_RET_RETRY) {
 		if (bo->htc->body_status != BS_NONE)
-			bo->doclose = SC_RESP_CLOSE;
+			bo->htc->doclose = SC_RESP_CLOSE;
 		if (bo->director_state != DIR_S_NULL)
 			VDI_Finish(bo->wrk, bo);
 
@@ -494,14 +491,14 @@ vbf_fetch_body_helper(struct busyobj *bo)
 			VSLb(vfc->wrk->vsl, SLT_FetchError,
 			    "Pass delivery abandoned");
 			vfps = VFP_END;
-			bo->doclose = SC_RX_BODY;
+			bo->htc->doclose = SC_RX_BODY;
 			break;
 		}
 		AZ(vfc->failed);
 		l = est;
 		assert(l >= 0);
 		if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK) {
-			bo->doclose = SC_RX_BODY;
+			bo->htc->doclose = SC_RX_BODY;
 			break;
 		}
 
@@ -522,7 +519,7 @@ vbf_fetch_body_helper(struct busyobj *bo)
 	if (vfps == VFP_ERROR) {
 		AN(vfc->failed);
 		(void)VFP_Error(vfc, "Fetch pipeline failed to process");
-		bo->doclose = SC_RX_BODY;
+		bo->htc->doclose = SC_RX_BODY;
 	}
 
 	if (!bo->do_stream)
@@ -612,14 +609,14 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	if (VFP_Open(bo->vfc)) {
 		(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
-		bo->doclose = SC_RX_BODY;
+		bo->htc->doclose = SC_RX_BODY;
 		VDI_Finish(bo->wrk, bo);
 		return (F_STP_ERROR);
 	}
 
 	if (vbf_beresp2obj(bo)) {
 		(void)VFP_Error(bo->vfc, "Could not get storage");
-		bo->doclose = SC_RX_BODY;
+		bo->htc->doclose = SC_RX_BODY;
 		VDI_Finish(bo->wrk, bo);
 		return (F_STP_ERROR);
 	}
@@ -809,9 +806,10 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	    wrk->handling == VCL_RET_ABANDON) {
 		VSB_delete(synth_body);
 
-		bo->doclose = SC_RESP_CLOSE;
-		if (bo->director_state != DIR_S_NULL)
+		if (bo->director_state != DIR_S_NULL) {
+			bo->htc->doclose = SC_RESP_CLOSE;
 			VDI_Finish(bo->wrk, bo);
+		}
 
 		if (wrk->handling == VCL_RET_RETRY &&
 		    bo->retries++ < cache_param->max_retries)
@@ -897,7 +895,6 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 	THR_SetBusyobj(bo);
 	stp = F_STP_MKBEREQ;
-	assert(bo->doclose == SC_NULL);
 	assert(isnan(bo->t_first));
 	assert(isnan(bo->t_prev));
 	VSLb_ts_busyobj(bo, "Start", W_TIM_real(wrk));
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 12f18f1..cc63d92 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -135,7 +135,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
 		VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
 		VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
-		bo->doclose = SC_TX_ERROR;
+		htc->doclose = SC_TX_ERROR;
 		return (1);
 	}
 	VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
@@ -168,7 +168,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
 			VSLb(bo->vsl, SLT_FetchError,
 			    "http %sread error: overflow",
 			    first ? "first " : "");
-			bo->doclose = SC_RX_OVERFLOW;
+			htc->doclose = SC_RX_OVERFLOW;
 			return (-1);
 		}
 		if (hs == HTC_S_EOF) {
@@ -177,7 +177,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
 			    htc->rxbuf_e - htc->rxbuf_b;
 			VSLb(bo->vsl, SLT_FetchError, "http %sread error: EOF",
 			    first ? "first " : "");
-			bo->doclose = SC_RX_TIMEOUT;
+			htc->doclose = SC_RX_TIMEOUT;
 			return (first ? 1 : -1);
 		}
 		if (first) {
@@ -192,11 +192,11 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
 
 	if (HTTP1_DissectResponse(hp, htc)) {
 		VSLb(bo->vsl, SLT_FetchError, "http format error");
-		bo->doclose = SC_RX_JUNK;
+		htc->doclose = SC_RX_JUNK;
 		return (-1);
 	}
 
-	bo->doclose = http_DoConnection(hp);
+	htc->doclose = http_DoConnection(hp);
 
 	return (0);
 }
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index 45b137d..aba7445 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -172,7 +172,7 @@ V1P_Process(struct req *req, struct busyobj *bo, int fd)
 	VSLb_ts_req(req, "PipeSess", W_TIM_real(wrk));
 	pipecharge(req, &acct_pipe, bo->htc->vbc->backend->vsc);
 	SES_Close(req->sp, SC_TX_PIPE);
-	bo->doclose = SC_TX_PIPE;
+	bo->htc->doclose = SC_TX_PIPE;
 }
 
 /*--------------------------------------------------------------------*/



From fgsch at lodoss.net  Wed Jun 24 12:59:59 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 24 Jun 2015 14:59:59 +0200
Subject: [master] d27be69 Remove superflous debug
Message-ID: <E1Z7kHX-0008C5-Rd@project.varnish-software.com>


commit d27be69da69bf9ea032085f7810119941c8948a5
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Jun 23 18:47:07 2015 +0100

    Remove superflous debug
    
    This is always followed by VCL_call MISS

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 5ce8309..9c4861f 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -370,7 +370,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	AZ(req->objcore);
 	if (lr == HSH_MISS) {
 		/* Found nothing */
-		VSLb(req->vsl, SLT_Debug, "XXXX MISS");
 		AZ(oc);
 		AN(boc);
 		AN(boc->flags & OC_F_BUSY);



From fgsch at lodoss.net  Wed Jun 24 12:59:59 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 24 Jun 2015 14:59:59 +0200
Subject: [master] 5be40a6 Kill another SLT_Debug message
Message-ID: <E1Z7kHX-0008CB-Sm@project.varnish-software.com>


commit 5be40a69317f23d7df9582192c5c37e01b8b621e
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Jun 24 11:30:30 2015 +0100

    Kill another SLT_Debug message

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index df6bb96..3c408e3 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -53,8 +53,6 @@ VRT_error(VRT_CTX, unsigned code, const char *reason)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	VSLb(ctx->vsl, SLT_Debug, "VCL_error(%u, %s)", code,
-	    reason ?  reason : "(null)");
 	if (code < 100 || code > 999)
 		code = 503;
 	ctx->req->err_code = (uint16_t)code;



From fgsch at lodoss.net  Wed Jun 24 15:09:49 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 24 Jun 2015 17:09:49 +0200
Subject: [master] 48af2f5 Rename VRT_error() to VRT_synth() for consistency
Message-ID: <E1Z7mJB-0000C8-Te@project.varnish-software.com>


commit 48af2f5a7df8e8a4a3269d37159a0919d2c6a635
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Jun 24 14:27:57 2015 +0100

    Rename VRT_error() to VRT_synth() for consistency

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 3c408e3..814d4af 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -48,7 +48,7 @@ const void * const vrt_magic_string_unset = &vrt_magic_string_unset;
 /*--------------------------------------------------------------------*/
 
 void
-VRT_error(VRT_CTX, unsigned code, const char *reason)
+VRT_synth(VRT_CTX, unsigned code, const char *reason)
 {
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
diff --git a/include/vrt.h b/include/vrt.h
index 63d1a85..98168a3 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -253,7 +253,7 @@ void VRT_purge(VRT_CTX, double ttl, double grace, double keep);
 
 void VRT_count(VRT_CTX, unsigned);
 int VRT_rewrite(const char *, const char *);
-void VRT_error(VRT_CTX, unsigned, const char *);
+void VRT_synth(VRT_CTX, unsigned, const char *);
 int VRT_switch_config(const char *);
 
 struct http *VRT_selecthttp(VRT_CTX, enum gethdr_e);
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 1ec9d72..ea3e679 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -324,7 +324,7 @@ parse_return(struct vcc *tl)
 		}
 		ExpectErr(tl, '(');
 		vcc_NextToken(tl);
-		Fb(tl, 1, "VRT_error(ctx,\n");
+		Fb(tl, 1, "VRT_synth(ctx,\n");
 		tl->indent += INDENT;
 		vcc_Expr(tl, INT);
 		ERRCHK(tl);



From fgsch at lodoss.net  Wed Jun 24 16:04:23 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 24 Jun 2015 18:04:23 +0200
Subject: [master] 04ad2c8 Grammar
Message-ID: <E1Z7n9z-0003Se-L4@project.varnish-software.com>


commit 04ad2c8f7394b782d37780e190e388fc02de1873
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Jun 24 16:50:48 2015 +0100

    Grammar

diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index 60a0c4e..9c07c55 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -435,8 +435,8 @@ struct parspec mgt_parspec[] = {
 		NULL },
 	{ "vcl_cooldown", tweak_timeout, &mgt_param.vcl_cooldown,
 		"0", NULL,
-		"How long time a VCL is kept warm after being replaced as the"
-		" active VCL.  (Granularity approximately 30 seconds.)",
+		"How long a VCL is kept warm after being replaced as the"
+		" active VCL (granularity approximately 30 seconds).",
 		0,
 		"600", "seconds" },
 	{ "vcc_err_unref", tweak_bool, &mgt_vcc_err_unref,



From phk at FreeBSD.org  Wed Jun 24 20:45:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 22:45:12 +0200
Subject: [master] d486144 Point the vbc at the cond-var,
 rather than the worker while it's stolen. This should make it less
 tempting to do illadvised things.
Message-ID: <E1Z7rXk-0007DF-7o@project.varnish-software.com>


commit d486144aaf900f44d394923817a3a21a5682a186
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 16:35:52 2015 +0000

    Point the vbc at the cond-var, rather than the worker while it's
    stolen.  This should make it less tempting to do illadvised things.

diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 37ce9ce..bd812ab 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -100,7 +100,7 @@ struct vbc {
 	struct tcp_pool		*tcp_pool;
 
 	struct backend		*backend;
-	struct worker		*wrk;
+	pthread_cond_t		*cond;
 };
 
 /* cache_backend.c */
@@ -128,5 +128,5 @@ void VBT_Recycle(const struct worker *, struct tcp_pool *, struct vbc **);
 void VBT_Close(struct tcp_pool *tp, struct vbc **vbc);
 struct vbc *VBT_Get(struct tcp_pool *, double tmo, struct backend *,
     struct worker *);
-void VBT_Wait(struct worker *wrk, const struct vbc *vbc);
+void VBT_Wait(struct worker *, struct vbc *);
 
diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index 9a658dd..a307e71 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -96,8 +96,8 @@ tcp_handle(struct waited *w, enum wait_event ev, double now)
 		vbc->state = VBC_STATE_USED;
 		VTAILQ_REMOVE(&tp->connlist, vbc, list);
 		CHECK_OBJ_NOTNULL(vbc->backend, BACKEND_MAGIC);
-		CHECK_OBJ_NOTNULL(vbc->wrk, WORKER_MAGIC);
-		AZ(pthread_cond_signal(&vbc->wrk->cond));
+		AN(vbc->cond);
+		AZ(pthread_cond_signal(vbc->cond));
 		break;
 	case VBC_STATE_AVAIL:
 		VTCP_close(&vbc->fd);
@@ -366,7 +366,7 @@ VBT_Get(struct tcp_pool *tp, double tmo, struct backend *be, struct worker *wrk)
 		VSC_C_main->backend_reuse += 1;
 		vbc->state = VBC_STATE_STOLEN;
 		vbc->backend = be;
-		vbc->wrk = wrk;
+		vbc->cond = &wrk->cond;
 	}
 	tp->n_used++;			// Opening mostly works
 	Lck_Unlock(&tp->mtx);
@@ -395,7 +395,7 @@ VBT_Get(struct tcp_pool *tp, double tmo, struct backend *be, struct worker *wrk)
  */
 
 void
-VBT_Wait(struct worker *wrk, const struct vbc *vbc)
+VBT_Wait(struct worker *wrk, struct vbc *vbc)
 {
 	struct tcp_pool *tp;
 
@@ -404,10 +404,11 @@ VBT_Wait(struct worker *wrk, const struct vbc *vbc)
 	CHECK_OBJ_NOTNULL(vbc->backend, BACKEND_MAGIC);
 	tp = vbc->tcp_pool;
 	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
-	assert(vbc->wrk == wrk);
+	assert(vbc->cond == &wrk->cond);
 	Lck_Lock(&tp->mtx);
 	while (vbc->state == VBC_STATE_STOLEN)
 		AZ(Lck_CondWait(&wrk->cond, &tp->mtx, 0));
 	assert(vbc->state == VBC_STATE_USED);
+	vbc->cond = NULL;
 	Lck_Unlock(&tp->mtx);
 }



From phk at FreeBSD.org  Wed Jun 24 20:45:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 22:45:12 +0200
Subject: [master] 5713c11 Eliminate the only actual use of vbc->backend,
 in preparation for getting rid of it entirely.
Message-ID: <E1Z7rXk-0007DL-9A@project.varnish-software.com>


commit 5713c119032b1f948c076ee3d2602643b2e972a9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 17:06:46 2015 +0000

    Eliminate the only actual use of vbc->backend, in preparation for getting
    rid of it entirely.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8883de1..490c11b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -723,7 +723,7 @@ vtr_deliver_f V1D_Deliver;
 
 /* cache_http1_pipe.c */
 void V1P_Init(void);
-void V1P_Process(struct req *req, struct busyobj *bo, int fd);
+void V1P_Process(struct req *, struct busyobj *, int fd, struct VSC_C_vbe *);
 
 /* cache_req_body.c */
 int VRB_Ignore(struct req *req);
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index aba7445..04ee8dd 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -76,6 +76,7 @@ static void
 pipecharge(struct req *req, const struct acct_pipe *a, struct VSC_C_vbe *b)
 {
 
+	AN(b);
 	VSLb(req->vsl, SLT_PipeAcct, "%ju %ju %ju %ju",
 	    (uintmax_t)a->req,
 	    (uintmax_t)a->bereq,
@@ -86,16 +87,14 @@ pipecharge(struct req *req, const struct acct_pipe *a, struct VSC_C_vbe *b)
 	VSC_C_main->s_pipe_hdrbytes += a->req;
 	VSC_C_main->s_pipe_in += a->in;
 	VSC_C_main->s_pipe_out += a->out;
-	if (b != NULL) {
-		b->pipe_hdrbytes += a->bereq;
-		b->pipe_out += a->in;
-		b->pipe_in += a->out;
-	}
+	b->pipe_hdrbytes += a->bereq;
+	b->pipe_out += a->in;
+	b->pipe_in += a->out;
 	Lck_Unlock(&pipestat_mtx);
 }
 
 void
-V1P_Process(struct req *req, struct busyobj *bo, int fd)
+V1P_Process(struct req *req, struct busyobj *bo, int fd, struct VSC_C_vbe *vsc)
 {
 	struct worker *wrk;
 	struct pollfd fds[2];
@@ -170,7 +169,7 @@ V1P_Process(struct req *req, struct busyobj *bo, int fd)
 		}
 	}
 	VSLb_ts_req(req, "PipeSess", W_TIM_real(wrk));
-	pipecharge(req, &acct_pipe, bo->htc->vbc->backend->vsc);
+	pipecharge(req, &acct_pipe, vsc);
 	SES_Close(req->sp, SC_TX_PIPE);
 	bo->htc->doclose = SC_TX_PIPE;
 }



From phk at FreeBSD.org  Wed Jun 24 20:45:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 22:45:12 +0200
Subject: [master] 7c898d1 Forgotten in previous commit.
Message-ID: <E1Z7rXk-0007DU-A9@project.varnish-software.com>


commit 7c898d1de9ed602c2f84d9d5fad15a278f915c0a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 17:08:22 2015 +0000

    Forgotten in previous commit.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 66c79d8..c2668a9 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -77,19 +77,17 @@ VBE_Healthy(const struct backend *backend, double *changed)
  * Get a connection to the backend
  */
 
-static int __match_proto__(vdi_getfd_f)
-vbe_dir_getfd(struct worker *wrk, const struct director *d, struct busyobj *bo)
+static int
+vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo)
 {
 	struct vbc *vc;
-	struct backend *bp;
 	double tmod;
 	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
 	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
+	CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC);
 	AN(bp->vsc);
 
 	if (!VBE_Healthy(bp, NULL)) {
@@ -210,7 +208,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
 	do {
-		i = vbe_dir_getfd(wrk, d, bo);
+		i = vbe_dir_getfd(wrk, bp, bo);
 		if (i < 0) {
 			VSLb(bo->vsl, SLT_FetchError, "no backend connection");
 			return (-1);
@@ -277,17 +275,19 @@ static void
 vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 {
 	int i;
+	struct backend *bp;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
-	i = vbe_dir_getfd(req->wrk, d, bo);
+	i = vbe_dir_getfd(req->wrk, bp, bo);
 	if (i < 0) {
 		VSLb(bo->vsl, SLT_FetchError, "no backend connection");
 		SES_Close(req->sp, SC_RX_TIMEOUT);
 		return;
 	} else {
-		V1P_Process(req, bo, i);
+		V1P_Process(req, bo, i, bp->vsc);
 		vbe_dir_finish(d, req->wrk, bo);
 	}
 }



From phk at FreeBSD.org  Wed Jun 24 20:45:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 22:45:12 +0200
Subject: [master] fdf06b4 Let directors and backends do their own panic dumps.
Message-ID: <E1Z7rXk-0007De-Bq@project.varnish-software.com>


commit fdf06b4b3996dc495c27fc873e214029238a0621
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 17:40:30 2015 +0000

    Let directors and backends do their own panic dumps.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index c2668a9..ca40b75 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -277,6 +277,7 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 	int i;
 	struct backend *bp;
 
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
@@ -294,6 +295,36 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 
 /*--------------------------------------------------------------------*/
 
+static void
+vbe_panic(const struct director *d, struct vsb *vsb)
+{
+	struct backend *bp;
+
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
+
+	VSB_printf(vsb, "      display_name = %s\n", bp->display_name);
+	if (bp->ipv4_addr != NULL)
+		VSB_printf(vsb, "      ipv4 = %s\n", bp->ipv4_addr);
+	if (bp->ipv6_addr != NULL)
+		VSB_printf(vsb, "      ipv6 = %s\n", bp->ipv6_addr);
+	VSB_printf(vsb, "      port = %s\n", bp->port);
+	VSB_printf(vsb, "      hosthdr = %s\n", bp->hosthdr);
+	VSB_printf(vsb, "      health=%s, admin_health=",
+	    bp->healthy ? "healthy" : "sick");
+	if (bp->admin_health == ah_probe)
+		VSB_printf(vsb, "probe");
+	else if (bp->admin_health == ah_sick)
+		VSB_printf(vsb, "sick");
+	else if (bp->admin_health == ah_healthy)
+		VSB_printf(vsb, "healthy");
+	else
+		VSB_printf(vsb, "*invalid*");
+	VSB_printf(vsb, ", changed=%.1f\n", bp->health_changed);
+}
+
+/*--------------------------------------------------------------------*/
+
 void
 VBE_fill_director(struct backend *be)
 {
@@ -312,4 +343,5 @@ VBE_fill_director(struct backend *be)
 	d->getbody = vbe_dir_getbody;
 	d->getip = vbe_dir_getip;
 	d->finish = vbe_dir_finish;
+	d->panic = vbe_panic;
 }
diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c
index 49a0405..f7e0479 100644
--- a/bin/varnishd/cache/cache_director.c
+++ b/bin/varnishd/cache/cache_director.c
@@ -178,3 +178,19 @@ VDI_Healthy(const struct director *d, const struct busyobj *bo)
 		return (1);
 	return (d->healthy(d, bo, NULL));
 }
+
+/* Dump panic info -----------------------------------------------------
+ */
+
+void
+VDI_Panic(const struct director *d, struct vsb *vsb, const char *nm)
+{
+	if (d == NULL)
+		return;
+	VSB_printf(vsb, "    %s = %p {\n", nm, d);
+	VSB_printf(vsb, "      vcl_name = %s\n", d->vcl_name);
+	VSB_printf(vsb, "      name = %s\n", d->name);
+	if (d->panic != NULL)
+		d->panic(d, vsb);
+	VSB_printf(vsb, "    }\n");
+}
diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h
index cf7af4b..cf20dc1 100644
--- a/bin/varnishd/cache/cache_director.h
+++ b/bin/varnishd/cache/cache_director.h
@@ -61,6 +61,8 @@ typedef void vdi_finish_f(const struct director *, struct worker *,
 typedef void vdi_http1pipe_f(const struct director *, struct req *,
     struct busyobj *);
 
+typedef void vdi_panic_f(const struct director *, struct vsb *);
+
 struct director {
 	unsigned		magic;
 #define DIRECTOR_MAGIC		0x3336351d
@@ -73,6 +75,7 @@ struct director {
 	vdi_getbody_f		*getbody;
 	vdi_getip_f		*getip;
 	vdi_finish_f		*finish;
+	vdi_panic_f		*panic;
 	void			*priv;
 	const void		*priv2;
 };
@@ -88,3 +91,4 @@ void VDI_Finish(struct worker *wrk, struct busyobj *bo);
 int VDI_Http1Pipe(struct req *, struct busyobj *);
 
 int VDI_Healthy(const struct director *, const struct busyobj *);
+void VDI_Panic(const struct director *, struct vsb *, const char *nm);
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 8412d30..bfe3e70 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -46,7 +46,6 @@
 
 #include "vrt.h"
 #include "cache_director.h"
-#include "cache_backend.h"
 #include "storage/storage.h"
 #include "vcli_priv.h"
 
@@ -145,16 +144,12 @@ pan_ws(const struct ws *ws, int indent)
 /*--------------------------------------------------------------------*/
 
 static void
-pan_vbc(const struct vbc *vbc)
+pan_htc(const struct http_conn *htc)
 {
 
-	struct backend *be;
-
-	be = vbc->backend;
-
-	VSB_printf(pan_vsp, "  backend = %p fd = %d {\n", be, vbc->fd);
-	VSB_printf(pan_vsp, "    display_name = \"%s\",\n", be->display_name);
-	VSB_printf(pan_vsp, "  },\n");
+	VSB_printf(pan_vsp, "    http_conn = %p {\n", htc);
+	VSB_printf(pan_vsp, "      fd = %d,\n", htc->fd);
+	VSB_printf(pan_vsp, "    },\n");
 }
 
 /*--------------------------------------------------------------------*/
@@ -309,15 +304,15 @@ pan_busyobj(const struct busyobj *bo)
 
 	VSB_printf(pan_vsp, "  busyobj = %p {\n", bo);
 	pan_ws(bo->ws, 4);
-	VSB_printf(pan_vsp, "  refcnt = %u\n", bo->refcount);
-	VSB_printf(pan_vsp, "  retries = %d\n", bo->retries);
-	VSB_printf(pan_vsp, "  failed = %d\n", bo->vfc->failed);
-	VSB_printf(pan_vsp, "  state = %d\n", (int)bo->state);
-	VSB_printf(pan_vsp, "  flags = {\n");
-#define BO_FLAG(l, r, w, d) if(bo->l) VSB_printf(pan_vsp, "    " #l "\n");
+	VSB_printf(pan_vsp, "    refcnt = %u\n", bo->refcount);
+	VSB_printf(pan_vsp, "    retries = %d\n", bo->retries);
+	VSB_printf(pan_vsp, "    failed = %d\n", bo->vfc->failed);
+	VSB_printf(pan_vsp, "    state = %d\n", (int)bo->state);
+	VSB_printf(pan_vsp, "    flags = {\n");
+#define BO_FLAG(l, r, w, d) if(bo->l) VSB_printf(pan_vsp, "      " #l "\n");
 #include "tbl/bo_flags.h"
 #undef BO_FLAG
-	VSB_printf(pan_vsp, "  }\n");
+	VSB_printf(pan_vsp, "    }\n");
 
 	if (bo->htc != NULL) {
 		VSB_printf(pan_vsp, "    bodystatus = %d (%s),\n",
@@ -331,11 +326,14 @@ pan_busyobj(const struct busyobj *bo)
 			    vfe->vfp->name, (int)vfe->closed);
 		VSB_printf(pan_vsp, "\n");
 	}
-	VSB_printf(pan_vsp, "    },\n");
 
-	if (bo->htc != NULL && bo->htc->vbc != NULL &&
-	    VALID_OBJ(bo->htc->vbc, BACKEND_MAGIC))
-		pan_vbc(bo->htc->vbc);
+	if (VALID_OBJ(bo->htc, HTTP_CONN_MAGIC))
+		pan_htc(bo->htc);
+	VDI_Panic(bo->director_req, pan_vsp, "director_req");
+	if (bo->director_resp == bo->director_req)
+		VSB_printf(pan_vsp, "    director_resp = director_req\n");
+	else
+		VDI_Panic(bo->director_resp, pan_vsp, "director_resp");
 	if (bo->bereq != NULL && bo->bereq->ws != NULL)
 		pan_http("bereq", bo->bereq, 4);
 	if (bo->beresp != NULL && bo->beresp->ws != NULL)



From phk at FreeBSD.org  Wed Jun 24 20:45:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 22:45:12 +0200
Subject: [master] fec6289 Eliminate vbc->backend pointer
Message-ID: <E1Z7rXk-0007Dp-D7@project.varnish-software.com>


commit fec6289ee98075d5b659e92425752f4b74e6544e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 18:10:12 2015 +0000

    Eliminate vbc->backend pointer

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index ca40b75..7602155 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -119,7 +119,6 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo)
 	}
 
 	assert(vc->fd >= 0);
-	vc->backend = bp;
 	AN(vc->addr);
 
 	Lck_Lock(&bp->mtx);
@@ -173,8 +172,6 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 		return;
 	if (bo->htc->vbc->state != VBC_STATE_USED)
 		VBT_Wait(wrk, bo->htc->vbc);
-	CHECK_OBJ_NOTNULL(bo->htc->vbc->backend, BACKEND_MAGIC);
-	bo->htc->vbc->backend = NULL;
 	if (bo->htc->doclose != SC_NULL) {
 		VSLb(bo->vsl, SLT_BackendClose, "%d %s", bo->htc->vbc->fd,
 		    bp->display_name);
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index bd812ab..d7cb7d7 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -99,7 +99,6 @@ struct vbc {
 	struct waited		waited[1];
 	struct tcp_pool		*tcp_pool;
 
-	struct backend		*backend;
 	pthread_cond_t		*cond;
 };
 
@@ -126,7 +125,7 @@ void VBT_Rel(struct tcp_pool **tpp);
 int VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa);
 void VBT_Recycle(const struct worker *, struct tcp_pool *, struct vbc **);
 void VBT_Close(struct tcp_pool *tp, struct vbc **vbc);
-struct vbc *VBT_Get(struct tcp_pool *, double tmo, struct backend *,
+struct vbc *VBT_Get(struct tcp_pool *, double tmo, const struct backend *,
     struct worker *);
 void VBT_Wait(struct worker *, struct vbc *);
 
diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index a307e71..05fd232 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -95,7 +95,6 @@ tcp_handle(struct waited *w, enum wait_event ev, double now)
 	case VBC_STATE_STOLEN:
 		vbc->state = VBC_STATE_USED;
 		VTAILQ_REMOVE(&tp->connlist, vbc, list);
-		CHECK_OBJ_NOTNULL(vbc->backend, BACKEND_MAGIC);
 		AN(vbc->cond);
 		AZ(pthread_cond_signal(vbc->cond));
 		break;
@@ -262,7 +261,6 @@ VBT_Recycle(const struct worker *wrk, struct tcp_pool *tp, struct vbc **vbcp)
 
 	assert(vbc->state == VBC_STATE_USED);
 	assert(vbc->fd > 0);
-	AZ(vbc->backend);
 
 	Lck_Lock(&tp->mtx);
 	tp->n_used--;
@@ -321,7 +319,6 @@ VBT_Close(struct tcp_pool *tp, struct vbc **vbcp)
 
 	assert(vbc->state == VBC_STATE_USED);
 	assert(vbc->fd > 0);
-	AZ(vbc->backend);
 
 	Lck_Lock(&tp->mtx);
 	tp->n_used--;
@@ -344,7 +341,8 @@ VBT_Close(struct tcp_pool *tp, struct vbc **vbcp)
  */
 
 struct vbc *
-VBT_Get(struct tcp_pool *tp, double tmo, struct backend *be, struct worker *wrk)
+VBT_Get(struct tcp_pool *tp, double tmo, const struct backend *be,
+    struct worker *wrk)
 {
 	struct vbc *vbc;
 
@@ -355,7 +353,7 @@ VBT_Get(struct tcp_pool *tp, double tmo, struct backend *be, struct worker *wrk)
 	Lck_Lock(&tp->mtx);
 	vbc = VTAILQ_FIRST(&tp->connlist);
 	CHECK_OBJ_ORNULL(vbc, VBC_MAGIC);
-	if (vbc == NULL || vbc->backend != NULL)
+	if (vbc == NULL || vbc->state == VBC_STATE_STOLEN)
 		vbc = NULL;
 	else {
 		assert(vbc->tcp_pool == tp);
@@ -365,7 +363,6 @@ VBT_Get(struct tcp_pool *tp, double tmo, struct backend *be, struct worker *wrk)
 		tp->n_conn--;
 		VSC_C_main->backend_reuse += 1;
 		vbc->state = VBC_STATE_STOLEN;
-		vbc->backend = be;
 		vbc->cond = &wrk->cond;
 	}
 	tp->n_used++;			// Opening mostly works
@@ -379,7 +376,6 @@ VBT_Get(struct tcp_pool *tp, double tmo, struct backend *be, struct worker *wrk)
 	INIT_OBJ(vbc->waited, WAITED_MAGIC);
 	vbc->state = VBC_STATE_USED;
 	vbc->tcp_pool = tp;
-	vbc->backend = be;
 	vbc->fd = VBT_Open(tp, tmo, &vbc->addr);
 	if (vbc->fd < 0)
 		FREE_OBJ(vbc);
@@ -401,7 +397,6 @@ VBT_Wait(struct worker *wrk, struct vbc *vbc)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(vbc->backend, BACKEND_MAGIC);
 	tp = vbc->tcp_pool;
 	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
 	assert(vbc->cond == &wrk->cond);



From phk at FreeBSD.org  Wed Jun 24 20:45:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 22:45:12 +0200
Subject: [master] 2ea02cb Segregate HTTP1 protocol stuff in its own include
 file.
Message-ID: <E1Z7rXk-0007E0-Eb@project.varnish-software.com>


commit 2ea02cb47c42e977255279c8f793f1a1a034c5c5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 18:32:32 2015 +0000

    Segregate HTTP1 protocol stuff in its own include file.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index e82770a..dc20242 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -103,6 +103,7 @@ noinst_HEADERS = \
 	cache/cache_pool.h \
 	common/heritage.h \
 	hash/hash_slinger.h \
+	http1/cache_http1.h \
 	mgt/mgt.h \
 	mgt/mgt_cli.h \
 	mgt/mgt_param.h \
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 490c11b..a9c04c9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -707,27 +707,10 @@ ssize_t VBO_waitlen(struct worker *, struct busyobj *, ssize_t l);
 void VBO_setstate(struct busyobj *bo, enum busyobj_state_e next);
 void VBO_waitstate(struct busyobj *bo, enum busyobj_state_e want);
 
-
-/* cache_http1_fetch.c [V1F] */
-int V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host);
-void V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc);
-
-/* cache_http1_fsm.c [HTTP1] */
-typedef int (req_body_iter_f)(struct req *, void *priv, void *ptr, size_t);
-void HTTP1_Session(struct worker *, struct req *);
-extern const int HTTP1_Req[3];
-extern const int HTTP1_Resp[3];
-
-/* cache_http1_deliver.c */
-vtr_deliver_f V1D_Deliver;
-
-/* cache_http1_pipe.c */
-void V1P_Init(void);
-void V1P_Process(struct req *, struct busyobj *, int fd, struct VSC_C_vbe *);
-
 /* cache_req_body.c */
 int VRB_Ignore(struct req *req);
 ssize_t VRB_Cache(struct req *req, ssize_t maxsize);
+typedef int (req_body_iter_f)(struct req *, void *priv, void *ptr, size_t);
 ssize_t VRB_Iterate(struct req *req, req_body_iter_f *func, void *priv);
 void VRB_Free(struct req *req);
 
@@ -958,14 +941,6 @@ int Pool_TrySumstat(struct worker *wrk);
 void Pool_PurgeStat(unsigned nobj);
 int Pool_Task_Any(struct pool_task *task, enum task_how how);
 
-#define V1L_IsReleased(w)	((w)->v1l == NULL)
-void V1L_Chunked(const struct worker *w);
-void V1L_EndChunk(const struct worker *w);
-void V1L_Reserve(struct worker *, struct ws *, int *fd, struct vsl_log *, double t0);
-unsigned V1L_Flush(const struct worker *w);
-unsigned V1L_FlushRelease(struct worker *w);
-size_t V1L_Write(const struct worker *w, const void *ptr, ssize_t len);
-
 /* cache_proxy.c [VPX] */
 task_func_t VPX_Proto_Sess;
 
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 7602155..46fe79c 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -41,6 +41,7 @@
 
 #include "cache_director.h"
 #include "cache_backend.h"
+#include "http1/cache_http1.h"
 
 #define FIND_TMO(tmx, dst, bo, be)					\
 	do {								\
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index ce83b99..0d7ba9f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -621,9 +621,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		return (F_STP_ERROR);
 	}
 
-	assert(V1L_IsReleased(wrk));
-
-
 	if (bo->do_esi)
 		ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_ESIPROC, 1);
 
@@ -916,7 +913,6 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 			WRONG("Illegal fetch_step");
 		}
 	}
-	assert(V1L_IsReleased(wrk));
 
 	assert(bo->director_state == DIR_S_NULL);
 
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index da9c297..cb5175a 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -33,6 +33,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "http1/cache_http1.h"
 #include "common/heritage.h"
 
 #include "vcli_priv.h"
diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c
index c3965ce..a353c81 100644
--- a/bin/varnishd/cache/cache_req_body.c
+++ b/bin/varnishd/cache/cache_req_body.c
@@ -33,6 +33,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "http1/cache_http1.h"
 #include "cache_filter.h"
 #include "vtim.h"
 #include "hash/hash_slinger.h"
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index e19c0b9..f3ab04a 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -47,6 +47,7 @@
 
 #include "cache.h"
 #include "cache_pool.h"
+#include "http1/cache_http1.h"
 
 #include "vsa.h"
 #include "vtcp.h"
diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h
new file mode 100644
index 0000000..dcd4cb0
--- /dev/null
+++ b/bin/varnishd/http1/cache_http1.h
@@ -0,0 +1,54 @@
+/*-
+ * Copyright (c) 2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/* cache_http1_fetch.c [V1F] */
+int V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host);
+void V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc);
+
+/* cache_http1_fsm.c [HTTP1] */
+void HTTP1_Session(struct worker *, struct req *);
+extern const int HTTP1_Req[3];
+extern const int HTTP1_Resp[3];
+
+/* cache_http1_deliver.c */
+vtr_deliver_f V1D_Deliver;
+
+/* cache_http1_pipe.c */
+void V1P_Init(void);
+void V1P_Process(struct req *, struct busyobj *, int fd, struct VSC_C_vbe *);
+
+/* cache_http1_line.c */
+void V1L_Chunked(const struct worker *w);
+void V1L_EndChunk(const struct worker *w);
+void V1L_Reserve(struct worker *, struct ws *, int *fd, struct vsl_log *,
+    double t0);
+unsigned V1L_Flush(const struct worker *w);
+unsigned V1L_FlushRelease(struct worker *w);
+size_t V1L_Write(const struct worker *w, const void *ptr, ssize_t len);
+
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index 710dee2..f0762df 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -31,6 +31,7 @@
 
 #include "cache/cache.h"
 #include "cache/cache_filter.h"
+#include "cache_http1.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index cc63d92..6223b15 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -44,6 +44,7 @@
 
 #include "cache/cache_director.h"
 #include "cache/cache_backend.h"
+#include "cache_http1.h"
 
 /*--------------------------------------------------------------------
  * Pass the request body to the backend
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index 8954061..103ba3b 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -38,6 +38,7 @@
 #include <stdlib.h>
 
 #include "cache/cache.h"
+#include "cache_http1.h"
 #include "hash/hash_slinger.h"
 
 #include "vtcp.h"
diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c
index a71e163..2bcaf6e 100644
--- a/bin/varnishd/http1/cache_http1_line.c
+++ b/bin/varnishd/http1/cache_http1_line.c
@@ -41,6 +41,7 @@
 #include <stdio.h>
 
 #include "cache/cache.h"
+#include "cache_http1.h"
 #include "vtim.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index 04ee8dd..b80141c 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -40,6 +40,7 @@
 #include "vtcp.h"
 #include "vtim.h"
 
+#include "cache_http1.h"
 #include "cache/cache_director.h"
 #include "cache/cache_backend.h"
 
diff --git a/bin/varnishd/http1/cache_http1_proto.c b/bin/varnishd/http1/cache_http1_proto.c
index fc20bae..933c232 100644
--- a/bin/varnishd/http1/cache_http1_proto.c
+++ b/bin/varnishd/http1/cache_http1_proto.c
@@ -46,6 +46,7 @@
 #include "config.h"
 
 #include "cache/cache.h"
+#include "cache_http1.h"
 
 #include "vct.h"
 
diff --git a/bin/varnishd/http1/cache_http1_vfp.c b/bin/varnishd/http1/cache_http1_vfp.c
index f620937..c63716d 100644
--- a/bin/varnishd/http1/cache_http1_vfp.c
+++ b/bin/varnishd/http1/cache_http1_vfp.c
@@ -40,6 +40,7 @@
 
 #include "cache/cache.h"
 #include "cache/cache_filter.h"
+#include "cache_http1.h"
 
 #include "vct.h"
 



From phk at FreeBSD.org  Wed Jun 24 20:45:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 22:45:12 +0200
Subject: [master] 275cf84 Split sending bereq into its own function,
 so the VBC/waiter dance does not pollute the http1 code.
Message-ID: <E1Z7rXk-0007EB-Fs@project.varnish-software.com>


commit 275cf8408db23dadcaef8c97d0be920529353921
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 18:48:29 2015 +0000

    Split sending bereq into its own function, so the VBC/waiter dance
    does not pollute the http1 code.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 46fe79c..effe64a 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -215,7 +215,16 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 		if (bo->htc->vbc->state != VBC_STATE_STOLEN)
 			extrachance = 0;
 
-		i = V1F_fetch_hdr(wrk, bo, bp->hosthdr);
+		i = V1F_SendReq(wrk, bo, bp->hosthdr);
+
+		if (bo->htc->vbc->state != VBC_STATE_USED)
+			VBT_Wait(wrk, bo->htc->vbc);
+
+		assert(bo->htc->vbc->state == VBC_STATE_USED);
+
+		if (!i)
+			i = V1F_FetchRespHdr(bo);
+
 		/*
 		 * If we recycled a backend connection, there is a finite chance
 		 * that the backend closed it before we got the bereq to it.
diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h
index dcd4cb0..8a80605 100644
--- a/bin/varnishd/http1/cache_http1.h
+++ b/bin/varnishd/http1/cache_http1.h
@@ -28,7 +28,8 @@
  */
 
 /* cache_http1_fetch.c [V1F] */
-int V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host);
+int V1F_SendReq(struct worker *, struct busyobj *, const char *def_host);
+int V1F_FetchRespHdr(struct busyobj *);
 void V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc);
 
 /* cache_http1_fsm.c [HTTP1] */
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 6223b15..15d4168 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -67,21 +67,18 @@ vbf_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
 }
 
 /*--------------------------------------------------------------------
- * Send request, and receive the HTTP protocol response, but not the
- * response body.
+ * Send request to backend, including any req.body
  *
  * Return value:
- *	-1 failure, not retryable
  *	 0 success
- *	 1 failure which can be retried.
+ *	 1 failure
  */
 
 int
-V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
+V1F_SendReq(struct worker *wrk, struct busyobj *bo, const char *def_host)
 {
 	struct http *hp;
-	enum htc_status_e hs;
-	int j, first;
+	int j;
 	ssize_t i;
 	struct http_conn *htc;
 	int do_chunked = 0;
@@ -140,15 +137,27 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
 		return (1);
 	}
 	VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
+	return (0);
+}
 
-	VSC_C_main->backend_req++;
+int
+V1F_FetchRespHdr(struct busyobj *bo)
+{
 
-	/* Receive response */
+	struct http *hp;
+	enum htc_status_e hs;
+	int first;
+	struct http_conn *htc;
 
-	if (htc->vbc->state != VBC_STATE_USED)
-		VBT_Wait(wrk, htc->vbc);
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
+	CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC);
 
-	assert(htc->vbc->state == VBC_STATE_USED);
+	htc = bo->htc;
+
+	VSC_C_main->backend_req++;
+
+	/* Receive response */
 
 	SES_RxInit(htc, bo->ws, cache_param->http_resp_size,
 	    cache_param->http_resp_hdr_len);



From phk at FreeBSD.org  Wed Jun 24 20:45:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 22:45:12 +0200
Subject: [master] 3c08479 Setting the default Host: header belongs in the
 backend code, not in the HTTP1 code.
Message-ID: <E1Z7rXk-0007EK-H3@project.varnish-software.com>


commit 3c08479d771b14aae887ff3a68d1b0d450fbb0aa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 19:07:01 2015 +0000

    Setting the default Host: header belongs in the backend code, not in
    the HTTP1 code.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index effe64a..5616b10 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -205,6 +205,14 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
+	/*
+	 * Now that we know our backend, we can set a default Host:
+	 * header if one is necessary.  This cannot be done in the VCL
+	 * because the backend may be chosen by a director.
+	 */
+	if (!http_GetHdr(bo->bereq, H_Host, NULL) && bp->hosthdr != NULL)
+		http_PrintfHeader(bo->bereq, "Host: %s", bp->hosthdr);
+
 	do {
 		i = vbe_dir_getfd(wrk, bp, bo);
 		if (i < 0) {
@@ -215,7 +223,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 		if (bo->htc->vbc->state != VBC_STATE_STOLEN)
 			extrachance = 0;
 
-		i = V1F_SendReq(wrk, bo, bp->hosthdr);
+		i = V1F_SendReq(wrk, bo);
 
 		if (bo->htc->vbc->state != VBC_STATE_USED)
 			VBT_Wait(wrk, bo->htc->vbc);
diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h
index 8a80605..9e5a45a 100644
--- a/bin/varnishd/http1/cache_http1.h
+++ b/bin/varnishd/http1/cache_http1.h
@@ -28,7 +28,7 @@
  */
 
 /* cache_http1_fetch.c [V1F] */
-int V1F_SendReq(struct worker *, struct busyobj *, const char *def_host);
+int V1F_SendReq(struct worker *, struct busyobj *);
 int V1F_FetchRespHdr(struct busyobj *);
 void V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc);
 
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 15d4168..2fe3c18 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -75,7 +75,7 @@ vbf_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
  */
 
 int
-V1F_SendReq(struct worker *wrk, struct busyobj *bo, const char *def_host)
+V1F_SendReq(struct worker *wrk, struct busyobj *bo)
 {
 	struct http *hp;
 	int j;
@@ -91,14 +91,6 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, const char *def_host)
 	htc = bo->htc;
 	hp = bo->bereq;
 
-	/*
-	 * Now that we know our backend, we can set a default Host:
-	 * header if one is necessary.  This cannot be done in the VCL
-	 * because the backend may be chosen by a director.
-	 */
-	if (!http_GetHdr(bo->bereq, H_Host, NULL) && def_host != NULL)
-		http_PrintfHeader(hp, "Host: %s", def_host);
-
 	if (bo->req != NULL &&
 	    bo->req->req_body_status == REQ_BODY_WITHOUT_LEN) {
 		http_PrintfHeader(hp, "Transfer-Encoding: chunked");



From phk at FreeBSD.org  Wed Jun 24 20:45:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 22:45:12 +0200
Subject: [master] dd01df1 Make the http1-pipe code use the same code to send
 the bereq,
 and properly send any cached req.body along (this didn't work before).
Message-ID: <E1Z7rXk-0007ET-I3@project.varnish-software.com>


commit dd01df1a9138d6a2204a1a8de4e0dc727d8fc488
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 20:10:25 2015 +0000

    Make the http1-pipe code use the same code to send the bereq, and
    properly send any cached req.body along (this didn't work before).
    
    This also makes struct vbc an entirely internal affair in cache_backend,
    as it (probably) should be.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 5616b10..9b144f5 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -38,6 +38,7 @@
 
 #include "vrt.h"
 #include "vtcp.h"
+#include "vtim.h"
 
 #include "cache_director.h"
 #include "cache_backend.h"
@@ -223,7 +224,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 		if (bo->htc->vbc->state != VBC_STATE_STOLEN)
 			extrachance = 0;
 
-		i = V1F_SendReq(wrk, bo);
+		i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes, 0);
 
 		if (bo->htc->vbc->state != VBC_STATE_USED)
 			VBT_Wait(wrk, bo->htc->vbc);
@@ -289,23 +290,39 @@ vbe_dir_getip(const struct director *d, struct worker *wrk,
 static void
 vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 {
-	int i;
+	int i, fd;
 	struct backend *bp;
+	struct v1p_acct v1a;
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
-	i = vbe_dir_getfd(req->wrk, bp, bo);
-	if (i < 0) {
+	memset(&v1a, 0, sizeof v1a);
+	req->res_mode = RES_PIPE;
+
+	fd = vbe_dir_getfd(req->wrk, bp, bo);
+	/* This is hackish... */
+	v1a.req = req->acct.req_hdrbytes;
+	req->acct.req_hdrbytes = 0;
+
+	if (fd < 0) {
 		VSLb(bo->vsl, SLT_FetchError, "no backend connection");
 		SES_Close(req->sp, SC_RX_TIMEOUT);
-		return;
 	} else {
-		V1P_Process(req, bo, i, bp->vsc);
+		i = V1F_SendReq(req->wrk, bo, &v1a.bereq, 1);
+		VSLb_ts_req(req, "Pipe", W_TIM_real(req->wrk));
+		if (bo->htc->vbc->state == VBC_STATE_STOLEN)
+			VBT_Wait(req->wrk, bo->htc->vbc);
+		if (i == 0)
+			V1P_Process(req, fd, &v1a);
+		VSLb_ts_req(req, "PipeSess", W_TIM_real(req->wrk));
+		SES_Close(req->sp, SC_TX_PIPE);
+		bo->htc->doclose = SC_TX_PIPE;
 		vbe_dir_finish(d, req->wrk, bo);
 	}
+	V1P_Charge(req, &v1a, bp->vsc);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h
index 9e5a45a..66addc5 100644
--- a/bin/varnishd/http1/cache_http1.h
+++ b/bin/varnishd/http1/cache_http1.h
@@ -28,7 +28,8 @@
  */
 
 /* cache_http1_fetch.c [V1F] */
-int V1F_SendReq(struct worker *, struct busyobj *);
+int V1F_SendReq(struct worker *, struct busyobj *, uint64_t *ctr,
+    int onlycached);
 int V1F_FetchRespHdr(struct busyobj *);
 void V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc);
 
@@ -41,8 +42,16 @@ extern const int HTTP1_Resp[3];
 vtr_deliver_f V1D_Deliver;
 
 /* cache_http1_pipe.c */
+struct v1p_acct {
+	uint64_t        req;
+	uint64_t        bereq;
+	uint64_t        in;
+	uint64_t        out;
+};
+
 void V1P_Init(void);
-void V1P_Process(struct req *, struct busyobj *, int fd, struct VSC_C_vbe *);
+void V1P_Process(struct req *, int fd, struct v1p_acct *);
+void V1P_Charge(struct req *, const struct v1p_acct *, struct VSC_C_vbe *);
 
 /* cache_http1_line.c */
 void V1L_Chunked(const struct worker *w);
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 2fe3c18..04f3952 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -67,7 +67,7 @@ vbf_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
 }
 
 /*--------------------------------------------------------------------
- * Send request to backend, including any req.body
+ * Send request to backend, including any (cached) req.body
  *
  * Return value:
  *	 0 success
@@ -75,7 +75,8 @@ vbf_iter_req_body(struct req *req, void *priv, void *ptr, size_t l)
  */
 
 int
-V1F_SendReq(struct worker *wrk, struct busyobj *bo)
+V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr,
+    int onlycached)
 {
 	struct http *hp;
 	int j;
@@ -99,12 +100,13 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo)
 
 	(void)VTCP_blocking(htc->fd);	/* XXX: we should timeout instead */
 	V1L_Reserve(wrk, wrk->aws, &htc->fd, bo->vsl, bo->t_prev);
-	bo->acct.bereq_hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req);
+	*ctr += HTTP1_Write(wrk, hp, HTTP1_Req);
 
 	/* Deal with any message-body the request might (still) have */
 	i = 0;
 
-	if (bo->req != NULL) {
+	if (bo->req != NULL &&
+	    (bo->req->req_body_status == REQ_BODY_CACHED || !onlycached)) {
 		if (do_chunked)
 			V1L_Chunked(wrk);
 		i = VRB_Iterate(bo->req, vbf_iter_req_body, bo);
@@ -126,7 +128,7 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo)
 		    errno, strerror(errno));
 		VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
 		htc->doclose = SC_TX_ERROR;
-		return (1);
+		return (-1);
 	}
 	VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
 	return (0);
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index b80141c..6614a14 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -37,8 +37,6 @@
 #include "cache/cache.h"
 
 #include "vrt.h"
-#include "vtcp.h"
-#include "vtim.h"
 
 #include "cache_http1.h"
 #include "cache/cache_director.h"
@@ -46,13 +44,6 @@
 
 static struct lock pipestat_mtx;
 
-struct acct_pipe {
-	uint64_t	req;
-	uint64_t	bereq;
-	uint64_t	in;
-	uint64_t	out;
-};
-
 static int
 rdf(int fd0, int fd1, uint64_t *pcnt)
 {
@@ -73,8 +64,8 @@ rdf(int fd0, int fd1, uint64_t *pcnt)
 	return (0);
 }
 
-static void
-pipecharge(struct req *req, const struct acct_pipe *a, struct VSC_C_vbe *b)
+void
+V1P_Charge(struct req *req, const struct v1p_acct *a, struct VSC_C_vbe *b)
 {
 
 	AN(b);
@@ -95,84 +86,56 @@ pipecharge(struct req *req, const struct acct_pipe *a, struct VSC_C_vbe *b)
 }
 
 void
-V1P_Process(struct req *req, struct busyobj *bo, int fd, struct VSC_C_vbe *vsc)
+V1P_Process(struct req *req, int fd, struct v1p_acct *v1a)
 {
-	struct worker *wrk;
 	struct pollfd fds[2];
-	int i;
-	struct acct_pipe acct_pipe;
+	int i, j;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC);
-	wrk = req->wrk;
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	assert(fd > 0);
 
-	req->res_mode = RES_PIPE;
-
-	memset(&acct_pipe, 0, sizeof acct_pipe);
-	acct_pipe.req = req->acct.req_hdrbytes;
-	req->acct.req_hdrbytes = 0;
-
-	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
-	CHECK_OBJ_NOTNULL(bo->htc->vbc, VBC_MAGIC);
-	bo->wrk = req->wrk;
-	bo->director_state = DIR_S_BODY;
-	(void)VTCP_blocking(fd);
-
-	V1L_Reserve(wrk, wrk->aws, &fd, bo->vsl, req->t_req);
-	acct_pipe.bereq += HTTP1_Write(wrk, bo->bereq, HTTP1_Req);
-
-	if (req->htc->pipeline_b != NULL)
-		acct_pipe.in += V1L_Write(wrk, req->htc->pipeline_b,
+	if (req->htc->pipeline_b != NULL) {
+		j = write(fd,  req->htc->pipeline_b,
 		    req->htc->pipeline_e - req->htc->pipeline_b);
-
-	i = V1L_FlushRelease(wrk);
-
-	VSLb_ts_req(req, "Pipe", W_TIM_real(wrk));
-
-	if (i == 0) {
-		if (bo->htc->vbc->state == VBC_STATE_STOLEN)
-			VBT_Wait(req->wrk, bo->htc->vbc);
-
-		memset(fds, 0, sizeof fds);
-		fds[0].fd = fd;
-		fds[0].events = POLLIN | POLLERR;
-		fds[1].fd = req->sp->fd;
-		fds[1].events = POLLIN | POLLERR;
-
-		while (fds[0].fd > -1 || fds[1].fd > -1) {
-			fds[0].revents = 0;
-			fds[1].revents = 0;
-			i = poll(fds, 2,
-			    (int)(cache_param->pipe_timeout * 1e3));
-			if (i < 1)
+		if (j < 0)
+			return;
+		req->htc->pipeline_b = NULL;
+		req->htc->pipeline_e = NULL;
+		v1a->in += j;
+	}
+	memset(fds, 0, sizeof fds);
+	fds[0].fd = fd;
+	fds[0].events = POLLIN | POLLERR;
+	fds[1].fd = req->sp->fd;
+	fds[1].events = POLLIN | POLLERR;
+
+	while (fds[0].fd > -1 || fds[1].fd > -1) {
+		fds[0].revents = 0;
+		fds[1].revents = 0;
+		i = poll(fds, 2,
+		    (int)(cache_param->pipe_timeout * 1e3));
+		if (i < 1)
+			break;
+		if (fds[0].revents &&
+		    rdf(fd, req->sp->fd, &v1a->out)) {
+			if (fds[1].fd == -1)
+				break;
+			(void)shutdown(fd, SHUT_RD);
+			(void)shutdown(req->sp->fd, SHUT_WR);
+			fds[0].events = 0;
+			fds[0].fd = -1;
+		}
+		if (fds[1].revents &&
+		    rdf(req->sp->fd, fd, &v1a->in)) {
+			if (fds[0].fd == -1)
 				break;
-			if (fds[0].revents &&
-			    rdf(fd, req->sp->fd, &acct_pipe.out)) {
-				if (fds[1].fd == -1)
-					break;
-				(void)shutdown(fd, SHUT_RD);
-				(void)shutdown(req->sp->fd, SHUT_WR);
-				fds[0].events = 0;
-				fds[0].fd = -1;
-			}
-			if (fds[1].revents &&
-			    rdf(req->sp->fd, fd, &acct_pipe.in)) {
-				if (fds[0].fd == -1)
-					break;
-				(void)shutdown(req->sp->fd, SHUT_RD);
-				(void)shutdown(fd, SHUT_WR);
-				fds[1].events = 0;
-				fds[1].fd = -1;
-			}
+			(void)shutdown(req->sp->fd, SHUT_RD);
+			(void)shutdown(fd, SHUT_WR);
+			fds[1].events = 0;
+			fds[1].fd = -1;
 		}
 	}
-	VSLb_ts_req(req, "PipeSess", W_TIM_real(wrk));
-	pipecharge(req, &acct_pipe, vsc);
-	SES_Close(req->sp, SC_TX_PIPE);
-	bo->htc->doclose = SC_TX_PIPE;
 }
 
 /*--------------------------------------------------------------------*/



From phk at FreeBSD.org  Wed Jun 24 20:45:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jun 2015 22:45:12 +0200
Subject: [master] 7043f69 Fully privatize struct vbc.
Message-ID: <E1Z7rXk-0007Eh-Jk@project.varnish-software.com>


commit 7043f697cfc001f7768ca5a17d14cfacb9320eaf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 24 20:42:26 2015 +0000

    Fully privatize struct vbc.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a9c04c9..b96d765 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -118,7 +118,6 @@ struct transport;
 struct req;
 struct sess;
 struct suckaddr;
-struct vbc;
 struct vrt_priv;
 struct vsb;
 struct waitinglist;
@@ -236,7 +235,7 @@ struct http_conn {
 	ssize_t			content_length;
 	enum body_status	body_status;
 	struct vfp_ctx		vfc[1];
-	struct vbc		*vbc;
+	void			*priv;
 
 	/* Timeouts */
 	double			first_byte_timeout;
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 9b144f5..7ed2673 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -79,7 +79,7 @@ VBE_Healthy(const struct backend *backend, double *changed)
  * Get a connection to the backend
  */
 
-static int
+static struct vbc *
 vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo)
 {
 	struct vbc *vc;
@@ -95,20 +95,20 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo)
 	if (!VBE_Healthy(bp, NULL)) {
 		// XXX: per backend stats ?
 		VSC_C_main->backend_unhealthy++;
-		return (-1);
+		return (NULL);
 	}
 
 	if (bp->max_connections > 0 && bp->n_conn >= bp->max_connections) {
 		// XXX: per backend stats ?
 		VSC_C_main->backend_busy++;
-		return (-1);
+		return (NULL);
 	}
 
 	AZ(bo->htc);
 	bo->htc = WS_Alloc(bo->ws, sizeof *bo->htc);
 	if (bo->htc == NULL)
 		/* XXX: counter ? */
-		return (-1);
+		return (NULL);
 	bo->htc->doclose = SC_NULL;
 
 	FIND_TMO(connect_timeout, tmod, bo, bp);
@@ -117,7 +117,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo)
 		// XXX: Per backend stats ?
 		VSC_C_main->backend_fail++;
 		bo->htc = NULL;
-		return (-1);
+		return (NULL);
 	}
 
 	assert(vc->fd >= 0);
@@ -136,13 +136,13 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo)
 
 	bp->vsc->req++;
 	INIT_OBJ(bo->htc, HTTP_CONN_MAGIC);
-	bo->htc->vbc = vc;
+	bo->htc->priv = vc;
 	bo->htc->fd = vc->fd;
 	FIND_TMO(first_byte_timeout,
 	    bo->htc->first_byte_timeout, bo, bp);
 	FIND_TMO(between_bytes_timeout,
 	    bo->htc->between_bytes_timeout, bo, bp);
-	return (vc->fd);
+	return (vc);
 }
 
 static unsigned __match_proto__(vdi_healthy_f)
@@ -162,6 +162,7 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
     struct busyobj *bo)
 {
 	struct backend *bp;
+	struct vbc *vbc;
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -169,27 +170,25 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
 	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
-	CHECK_OBJ_ORNULL(bo->htc->vbc, VBC_MAGIC);
-	if (bo->htc->vbc == NULL)
-		return;
-	if (bo->htc->vbc->state != VBC_STATE_USED)
-		VBT_Wait(wrk, bo->htc->vbc);
+	CAST_OBJ_NOTNULL(vbc, bo->htc->priv, VBC_MAGIC);
+	bo->htc->priv = NULL;
+	if (vbc->state != VBC_STATE_USED)
+		VBT_Wait(wrk, vbc);
 	if (bo->htc->doclose != SC_NULL) {
-		VSLb(bo->vsl, SLT_BackendClose, "%d %s", bo->htc->vbc->fd,
+		VSLb(bo->vsl, SLT_BackendClose, "%d %s", vbc->fd,
 		    bp->display_name);
-		VBT_Close(bp->tcp_pool, &bo->htc->vbc);
+		VBT_Close(bp->tcp_pool, &vbc);
 		Lck_Lock(&bp->mtx);
 	} else {
-		VSLb(bo->vsl, SLT_BackendReuse, "%d %s", bo->htc->vbc->fd,
+		VSLb(bo->vsl, SLT_BackendReuse, "%d %s", vbc->fd,
 		    bp->display_name);
 		Lck_Lock(&bp->mtx);
 		VSC_C_main->backend_recycle++;
-		VBT_Recycle(wrk, bp->tcp_pool, &bo->htc->vbc);
+		VBT_Recycle(wrk, bp->tcp_pool, &vbc);
 	}
 #define ACCT(foo)	bp->vsc->foo += bo->acct.foo;
 #include "tbl/acct_fields_bereq.h"
 #undef ACCT
-	bo->htc->vbc = NULL;
 	Lck_Unlock(&bp->mtx);
 	bo->htc = NULL;
 }
@@ -200,6 +199,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 {
 	int i, extrachance = 1;
 	struct backend *bp;
+	struct vbc *vbc;
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -215,34 +215,34 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 		http_PrintfHeader(bo->bereq, "Host: %s", bp->hosthdr);
 
 	do {
-		i = vbe_dir_getfd(wrk, bp, bo);
-		if (i < 0) {
+		vbc = vbe_dir_getfd(wrk, bp, bo);
+		if (vbc == NULL) {
 			VSLb(bo->vsl, SLT_FetchError, "no backend connection");
 			return (-1);
 		}
 		AN(bo->htc);
-		if (bo->htc->vbc->state != VBC_STATE_STOLEN)
+		if (vbc->state != VBC_STATE_STOLEN)
 			extrachance = 0;
 
 		i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes, 0);
 
-		if (bo->htc->vbc->state != VBC_STATE_USED)
-			VBT_Wait(wrk, bo->htc->vbc);
+		if (vbc->state != VBC_STATE_USED)
+			VBT_Wait(wrk, vbc);
 
-		assert(bo->htc->vbc->state == VBC_STATE_USED);
+		assert(vbc->state == VBC_STATE_USED);
 
-		if (!i)
+		if (i == 0)
 			i = V1F_FetchRespHdr(bo);
+		if (i == 0) {
+			AN(bo->htc->priv);
+			return (0);
+		}
 
 		/*
 		 * If we recycled a backend connection, there is a finite chance
 		 * that the backend closed it before we got the bereq to it.
 		 * In that case do a single automatic retry if req.boy allows.
 		 */
-		if (i == 0) {
-			AN(bo->htc->vbc);
-			return (0);
-		}
 		vbe_dir_finish(d, wrk, bo);
 		AZ(bo->htc);
 		if (i < 0)
@@ -275,14 +275,15 @@ static const struct suckaddr * __match_proto__(vdi_getip_f)
 vbe_dir_getip(const struct director *d, struct worker *wrk,
     struct busyobj *bo)
 {
+	struct vbc *vbc;
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
-	CHECK_OBJ_NOTNULL(bo->htc->vbc, VBC_MAGIC);
+	CAST_OBJ_NOTNULL(vbc, bo->htc->priv, VBC_MAGIC);
 
-	return (bo->htc->vbc->addr);
+	return (vbc->addr);
 }
 
 /*--------------------------------------------------------------------*/
@@ -290,9 +291,10 @@ vbe_dir_getip(const struct director *d, struct worker *wrk,
 static void
 vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 {
-	int i, fd;
+	int i;
 	struct backend *bp;
 	struct v1p_acct v1a;
+	struct vbc *vbc;
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -300,23 +302,25 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
 	memset(&v1a, 0, sizeof v1a);
-	req->res_mode = RES_PIPE;
 
-	fd = vbe_dir_getfd(req->wrk, bp, bo);
 	/* This is hackish... */
 	v1a.req = req->acct.req_hdrbytes;
 	req->acct.req_hdrbytes = 0;
 
-	if (fd < 0) {
+	req->res_mode = RES_PIPE;
+
+	vbc = vbe_dir_getfd(req->wrk, bp, bo);
+
+	if (vbc == NULL) {
 		VSLb(bo->vsl, SLT_FetchError, "no backend connection");
 		SES_Close(req->sp, SC_RX_TIMEOUT);
 	} else {
 		i = V1F_SendReq(req->wrk, bo, &v1a.bereq, 1);
 		VSLb_ts_req(req, "Pipe", W_TIM_real(req->wrk));
-		if (bo->htc->vbc->state == VBC_STATE_STOLEN)
-			VBT_Wait(req->wrk, bo->htc->vbc);
+		if (vbc->state == VBC_STATE_STOLEN)
+			VBT_Wait(req->wrk, vbc);
 		if (i == 0)
-			V1P_Process(req, fd, &v1a);
+			V1P_Process(req, vbc->fd, &v1a);
 		VSLb_ts_req(req, "PipeSess", W_TIM_real(req->wrk));
 		SES_Close(req->sp, SC_TX_PIPE);
 		bo->htc->doclose = SC_TX_PIPE;
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index d7cb7d7..d1613f4 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -38,7 +38,6 @@
  */
 
 struct vbp_target;
-struct vbc;
 struct vrt_ctx;
 struct vrt_backend_probe;
 struct tcp_pool;
@@ -82,9 +81,10 @@ struct backend {
 	struct director		director[1];
 };
 
-/* -------------------------------------------------------------------*/
+/*---------------------------------------------------------------------
+ * Backend connection -- private to cache_backend*.c 
+ */
 
-/* Backend connection */
 struct vbc {
 	unsigned		magic;
 #define VBC_MAGIC		0x0c5e6592
@@ -102,6 +102,10 @@ struct vbc {
 	pthread_cond_t		*cond;
 };
 
+/*---------------------------------------------------------------------
+ * Prototypes
+ */
+
 /* cache_backend.c */
 void VBE_fill_director(struct backend *be);
 
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 04f3952..2a2099b 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -42,8 +42,6 @@
 
 #include "hash/hash_slinger.h"
 
-#include "cache/cache_director.h"
-#include "cache/cache_backend.h"
 #include "cache_http1.h"
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index 6614a14..85df545 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -39,8 +39,6 @@
 #include "vrt.h"
 
 #include "cache_http1.h"
-#include "cache/cache_director.h"
-#include "cache/cache_backend.h"
 
 static struct lock pipestat_mtx;
 



From nils.goroll at uplex.de  Fri Jun 26 12:38:38 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 26 Jun 2015 14:38:38 +0200
Subject: [master] 35cf781 Simplify vsl segment management,
 fixing spurious vsl overruns
Message-ID: <E1Z8Sty-0001ZG-8Y@project.varnish-software.com>


commit 35cf78131c2aaaf8227c31ba92eb577c5bea9855
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jun 15 12:47:04 2015 +0200

    Simplify vsl segment management, fixing spurious vsl overruns
    
    vsl sequence and segment updates didn't happen atomically, so
    vslc_vsm_check could report spurious overruns.
    
    Replace sequence and segment index with a counter (segment_n), which
    always increments (with overflow after UINT_MAX). The actual segment
    index will be segment_n % VSL_SEGMENTS. Overrun detection by
    calculation of the difference between two segment numbers becomes
    simple and safe because we only ever access/update a single integer.
    
    Update the shared memory log head.
    
    (struct VSLC_ptr).priv is now the the equivalent of segment_n from the
    reader side. It gets initialized once and is maintained independently.
    
    Patch prepared in collaboration with Martin Blix Grydeland
    <martin at varnish-software.com>
    
    Fixes: #1747

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 2388969..3566d53 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -47,9 +47,8 @@ static pthread_mutex_t vsm_mtx;
 static struct VSL_head		*vsl_head;
 static const uint32_t		*vsl_end;
 static uint32_t			*vsl_ptr;
-static unsigned			vsl_segment;
+static unsigned			vsl_segment_n;
 static ssize_t			vsl_segsize;
-static unsigned			vsl_seq;
 
 struct VSC_C_main       *VSC_C_main;
 
@@ -108,19 +107,16 @@ vsl_wrap(void)
 
 	assert(vsl_ptr >= vsl_head->log);
 	assert(vsl_ptr < vsl_end);
+	vsl_segment_n += VSL_SEGMENTS - (vsl_segment_n % VSL_SEGMENTS);
+	assert(vsl_segment_n % VSL_SEGMENTS == 0);
+	vsl_head->offset[0] = 0;
 	vsl_head->log[0] = VSL_ENDMARKER;
-	do
-		vsl_seq++;
-	while (vsl_seq == 0);
-	vsl_head->seq = vsl_seq;
-	vsl_head->segments[0] = 0;
 	VWMB();
 	if (vsl_ptr != vsl_head->log) {
 		*vsl_ptr = VSL_WRAPMARKER;
 		vsl_ptr = vsl_head->log;
 	}
-	vsl_segment = 0;
-	vsl_head->segment = vsl_segment;
+	vsl_head->segment_n = vsl_segment_n;
 	VSC_C_main->shm_cycles++;
 }
 
@@ -133,7 +129,6 @@ vsl_get(unsigned len, unsigned records, unsigned flushes)
 {
 	uint32_t *p;
 	int err;
-	unsigned old_segment;
 
 	err = pthread_mutex_trylock(&vsl_mtx);
 	if (err == EBUSY) {
@@ -160,21 +155,17 @@ vsl_get(unsigned len, unsigned records, unsigned flushes)
 
 	*vsl_ptr = VSL_ENDMARKER;
 
-	old_segment = vsl_segment;
-	while ((vsl_ptr - vsl_head->log) / vsl_segsize > vsl_segment) {
-		if (vsl_segment == VSL_SEGMENTS - 1)
-			break;
-		vsl_segment++;
-		vsl_head->segments[vsl_segment] = vsl_ptr - vsl_head->log;
-	}
-	if (old_segment != vsl_segment) {
-		/* Write memory barrier to ensure ENDMARKER and new table
-		   values are seen before new segment number */
-		VWMB();
-		vsl_head->segment = vsl_segment;
+	while ((vsl_ptr - vsl_head->log) / vsl_segsize >
+	    vsl_segment_n % VSL_SEGMENTS) {
+		vsl_segment_n++;
+		vsl_head->offset[vsl_segment_n % VSL_SEGMENTS] =
+		    vsl_ptr - vsl_head->log;
 	}
 
 	AZ(pthread_mutex_unlock(&vsl_mtx));
+	/* Implicit VWMB() in mutex op ensures ENDMARKER and new table
+	   values are seen before new segment number */
+	vsl_head->segment_n = vsl_segment_n;
 
 	return (p);
 }
@@ -458,29 +449,31 @@ VSM_Init(void)
 	int i;
 	pthread_t tp;
 
+	assert(UINT_MAX % VSL_SEGMENTS == VSL_SEGMENTS - 1);
+
 	AZ(pthread_mutex_init(&vsl_mtx, NULL));
 	AZ(pthread_mutex_init(&vsm_mtx, NULL));
 
 	vsl_head = VSM_Alloc(cache_param->vsl_space, VSL_CLASS, "", "");
 	AN(vsl_head);
-	vsl_end = vsl_head->log +
-	    (cache_param->vsl_space - sizeof *vsl_head) / sizeof *vsl_end;
-	vsl_segsize = (vsl_end - vsl_head->log) / VSL_SEGMENTS;
-
-	memset(vsl_head, 0, sizeof *vsl_head);
-	memcpy(vsl_head->marker, VSL_HEAD_MARKER, sizeof vsl_head->marker);
-	vsl_head->segments[0] = 0;
-	for (i = 1; i < VSL_SEGMENTS; i++)
-		vsl_head->segments[i] = -1;
+	vsl_segsize = (cache_param->vsl_space - sizeof *vsl_head) /
+	    sizeof *vsl_end / VSL_SEGMENTS;
+	vsl_end = vsl_head->log + vsl_segsize * VSL_SEGMENTS;
+	/* Make segment_n always overflow on first log wrap to make any
+	   problems with regard to readers on that event visible */
+	vsl_segment_n = 0U - VSL_SEGMENTS;
+	AZ(vsl_segment_n % VSL_SEGMENTS);
+	vsl_head->segment_n = vsl_segment_n;
 	vsl_ptr = vsl_head->log;
 	*vsl_ptr = VSL_ENDMARKER;
 
+	memset(vsl_head, 0, sizeof *vsl_head);
+	vsl_head->segsize = vsl_segsize;
+	vsl_head->offset[0] = 0;
+	for (i = 1; i < VSL_SEGMENTS; i++)
+		vsl_head->offset[i] = -1;
 	VWMB();
-	do
-		vsl_seq = random();
-	while (vsl_seq == 0);
-	vsl_head->seq = vsl_seq;
-	VWMB();
+	memcpy(vsl_head->marker, VSL_HEAD_MARKER, sizeof vsl_head->marker);
 
 	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,
 	    VSC_CLASS, VSC_type_main, "");
diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h
index e3046b2..15ed9af 100644
--- a/include/vapi/vsl_int.h
+++ b/include/vapi/vsl_int.h
@@ -40,7 +40,7 @@
 #define VAPI_VSL_INT_H_INCLUDED
 
 #define VSL_CLASS		"Log"
-#define VSL_SEGMENTS		8
+#define VSL_SEGMENTS		8U
 
 /*
  * Shared memory log format
diff --git a/include/vsl_priv.h b/include/vsl_priv.h
index d5643ed..0c780dc 100644
--- a/include/vsl_priv.h
+++ b/include/vsl_priv.h
@@ -39,32 +39,30 @@
 #include "vapi/vsm_int.h"
 
 #define VSL_CLASS		"Log"
-#define VSL_SEGMENTS		8
+#define VSL_SEGMENTS		8U	// power of two
 
 /*
  * Shared memory log format
  *
  * The segments array has index values providing safe entry points into
  * the log, where each element N gives the index of the first log record
- * in the Nth fraction of the log. An index value of -1 indicated that no
- * log records in this fraction exists.
+ * in the Nth segment of the log. An index value of -1 indicates that no
+ * log records in this segment exists.
  *
- * The segment member shows the current segment where Varnish is currently
- * appending log data.
- *
- * The seq member contains a non-zero seq number randomly initialized,
- * which increases whenever writing the log starts from the front.
+ * The segment_n member is incremented only, natively wrapping at
+ * UINT_MAX. When taken modulo VSL_SEGMENTS, it gives the current index
+ * into the offset array.
  *
  * The format of the actual log is in vapi/vsl_int.h
  *
  */
 
 struct VSL_head {
-#define VSL_HEAD_MARKER		"VSLHEAD0"	/* Incr. as version# */
+#define VSL_HEAD_MARKER		"VSLHEAD1"	/* Incr. as version# */
 	char			marker[VSM_MARKER_LEN];
-	volatile ssize_t	segments[VSL_SEGMENTS];
-	volatile unsigned	segment;	/* Current varnishd segment */
-	volatile unsigned	seq;		/* Non-zero seq number */
+	ssize_t			segsize;
+	unsigned		segment_n;
+	ssize_t			offset[VSL_SEGMENTS];
 	uint32_t		log[];
 };
 
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index ba6e7d2..b9bbcb3 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -42,6 +42,7 @@
 #include "vdef.h"
 #include "vas.h"
 #include "miniobj.h"
+#include "vmb.h"
 
 #include "vqueue.h"
 #include "vre.h"
@@ -66,7 +67,6 @@ struct vslc_vsm {
 
 	const struct VSL_head		*head;
 	const uint32_t			*end;
-	ssize_t				segsize;
 	struct VSLC_ptr			next;
 };
 
@@ -80,11 +80,21 @@ vslc_vsm_delete(const struct VSL_cursor *cursor)
 	FREE_OBJ(c);
 }
 
+/*
+ * We tolerate the fact that segment_n wraps around eventually: for the default
+ * vsl_space of 80MB and 8 segments, each segement is 10MB long, so we wrap
+ * roughly after 40 pebibytes (32bit) or 160 yobibytes (64bit) worth of vsl
+ * written.
+ *
+ * The vsm_check would fail if a vslc paused while this amount of data was
+ * written
+ */
+
 static int
 vslc_vsm_check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
 {
 	const struct vslc_vsm *c;
-	unsigned seqdiff, segment, segdiff;
+	unsigned dist;
 
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
 	assert(&c->cursor == cursor);
@@ -92,28 +102,12 @@ vslc_vsm_check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
 	if (ptr->ptr == NULL)
 		return (0);
 
-	/* Check sequence number */
-	seqdiff = c->head->seq - ptr->priv;
-	if (c->head->seq < ptr->priv)
-		/* Wrap around skips 0 */
-		seqdiff -= 1;
-	if (seqdiff > 1)
-		/* Too late */
-		return (0);
+	dist = c->head->segment_n - ptr->priv;
 
-	/* Check overrun */
-	segment = (ptr->ptr - c->head->log) / c->segsize;
-	if (segment >= VSL_SEGMENTS)
-		/* Rounding error spills to last segment */
-		segment = VSL_SEGMENTS - 1;
-	segdiff = (segment - c->head->segment) % VSL_SEGMENTS;
-	if (segdiff == 0 && seqdiff == 0)
-		/* In same segment, but close to tail */
-		return (2);
-	if (segdiff <= 2)
+	if (dist >= VSL_SEGMENTS - 2)
 		/* Too close to continue */
 		return (0);
-	if (segdiff <= 4)
+	if (dist >= VSL_SEGMENTS - 4)
 		/* Warning level */
 		return (1);
 	/* Safe */
@@ -131,11 +125,6 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 	assert(&c->cursor == cursor);
 	CHECK_OBJ_NOTNULL(c->vsm, VSM_MAGIC);
 
-	/* Assert pointers */
-	AN(c->next.ptr);
-	assert(c->next.ptr >= c->head->log);
-	assert(c->next.ptr < c->end);
-
 	i = vslc_vsm_check(&c->cursor, &c->next);
 	if (i <= 0)
 		/* Overrun */
@@ -149,9 +138,6 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 	}
 
 	while (1) {
-		assert(c->next.ptr >= c->head->log);
-		assert(c->next.ptr < c->end);
-		AN(c->head->seq);
 		t = *(volatile const uint32_t *)c->next.ptr;
 		AN(t);
 
@@ -159,17 +145,12 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 			/* Wrap around not possible at front */
 			assert(c->next.ptr != c->head->log);
 			c->next.ptr = c->head->log;
+			while (c->next.priv % VSL_SEGMENTS)
+				c->next.priv++;
 			continue;
 		}
 
 		if (t == VSL_ENDMARKER) {
-			if (c->next.ptr != c->head->log &&
-			    c->next.priv != c->head->seq) {
-				/* ENDMARKER not at front and seq wrapped */
-				/* XXX: assert on this? */
-				c->next.ptr = c->head->log;
-				continue;
-			}
 			if (c->options & VSL_COPT_TAILSTOP)
 				/* EOF */
 				return (-1);
@@ -177,11 +158,9 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 				return (0);
 		}
 
-		if (c->next.ptr == c->head->log)
-			c->next.priv = c->head->seq;
-
 		c->cursor.rec = c->next;
 		c->next.ptr = VSL_NEXT(c->next.ptr);
+
 		if (VSL_TAG(c->cursor.rec.ptr) == SLT__Batch) {
 			if (!(c->options & VSL_COPT_BATCH))
 				/* Skip the batch record */
@@ -191,6 +170,14 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 			c->next.ptr +=
 			    VSL_WORDS(VSL_BATCHLEN(c->cursor.rec.ptr));
 		}
+
+		while ((c->next.ptr - c->head->log) / c->head->segsize >
+		    c->next.priv % VSL_SEGMENTS)
+			c->next.priv++;
+
+		assert(c->next.ptr >= c->head->log);
+		assert(c->next.ptr < c->end);
+
 		return (1);
 	}
 }
@@ -199,25 +186,53 @@ static int
 vslc_vsm_reset(const struct VSL_cursor *cursor)
 {
 	struct vslc_vsm *c;
-	unsigned segment;
+	unsigned u, segment_n;
+	int i;
 
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
 	assert(&c->cursor == cursor);
-
-	/*
-	 * Starting (VSL_SEGMENTS - 3) behind varnishd. This way
-	 * even if varnishd wraps immediately, we'll still have a
-	 * full segment worth of log before the general constraint
-	 * of at least 2 segments apart will be broken
-	 */
-	segment = (c->head->segment + 3) % VSL_SEGMENTS;
-	if (c->head->segments[segment] < 0)
-		segment = 0;
-	assert(c->head->segments[segment] >= 0);
-	c->next.ptr = c->head->log + c->head->segments[segment];
-	c->next.priv = c->head->seq;
 	c->cursor.rec.ptr = NULL;
 
+	segment_n = c->head->segment_n;
+	VRMB();			/* Make sure offset table is not stale
+				   compared to segment_n */
+
+	if (c->options & VSL_COPT_TAIL) {
+		/* Start in the same segment varnishd currently is in and
+		   run forward until we see the end */
+		u = c->next.priv = segment_n;
+		assert(c->head->offset[c->next.priv % VSL_SEGMENTS] >= 0);
+		c->next.ptr = c->head->log +
+		    c->head->offset[c->next.priv % VSL_SEGMENTS];
+		do {
+			if (c->head->segment_n - u > 1) {
+				/* Give up if varnishd is moving faster
+				   than us */
+				return (-3); /* overrun */
+			}
+			i = vslc_vsm_next(&c->cursor);
+		} while (i == 1);
+		if (i)
+			return (i);
+	} else {
+		/* Starting (VSL_SEGMENTS - 3) behind varnishd. This way
+		 * even if varnishd advances segment_n immediately, we'll
+		 * still have a full segment worth of log before the
+		 * general constraint of at least 2 segments apart will be
+		 * broken.
+		 */
+		c->next.priv = segment_n - (VSL_SEGMENTS - 3);
+		while (c->head->offset[c->next.priv % VSL_SEGMENTS] < 0) {
+			/* seg 0 must be initialized */
+			assert(c->next.priv % VSL_SEGMENTS != 0);
+			c->next.priv++;
+		}
+		assert(c->head->offset[c->next.priv % VSL_SEGMENTS] >= 0);
+		c->next.ptr = c->head->log +
+		    c->head->offset[c->next.priv % VSL_SEGMENTS];
+	}
+	assert(c->next.ptr >= c->head->log);
+	assert(c->next.ptr < c->end);
 	return (0);
 }
 
@@ -235,28 +250,25 @@ VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, unsigned options)
 	struct vslc_vsm *c;
 	struct VSM_fantom vf;
 	struct VSL_head *head;
+	int i;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 	CHECK_OBJ_NOTNULL(vsm, VSM_MAGIC);
 
 	if (!VSM_Get(vsm, &vf, VSL_CLASS, "", "")) {
-		vsl_diag(vsl, "No VSL chunk found (child not started ?)\n");
+		(void)vsl_diag(vsl,
+		    "No VSL chunk found (child not started ?)\n");
 		return (NULL);
 	}
 
 	head = vf.b;
 	if (memcmp(head->marker, VSL_HEAD_MARKER, sizeof head->marker)) {
-		vsl_diag(vsl, "Not a VSL chunk\n");
-		return (NULL);
-	}
-	if (head->seq == 0) {
-		vsl_diag(vsl, "VSL chunk not initialized\n");
+		(void)vsl_diag(vsl, "Not a VSL chunk\n");
 		return (NULL);
 	}
-
 	ALLOC_OBJ(c, VSLC_VSM_MAGIC);
 	if (c == NULL) {
-		vsl_diag(vsl, "Out of memory\n");
+		(void)vsl_diag(vsl, "Out of memory\n");
 		return (NULL);
 	}
 	c->cursor.priv_tbl = &vslc_vsm_tbl;
@@ -266,19 +278,15 @@ VSL_CursorVSM(struct VSL_data *vsl, struct VSM_data *vsm, unsigned options)
 	c->vsm = vsm;
 	c->vf = vf;
 	c->head = head;
-	c->end = vf.e;
-	c->segsize = (c->end - c->head->log) / VSL_SEGMENTS;
+	c->end = c->head->log + c->head->segsize * VSL_SEGMENTS;
+	assert(c->end <= (const uint32_t *)vf.e);
 
-	if (c->options & VSL_COPT_TAIL) {
-		/* Locate tail of log */
-		c->next.ptr = c->head->log +
-		    c->head->segments[c->head->segment];
-		while (c->next.ptr < c->end &&
-		    *(volatile const uint32_t *)c->next.ptr != VSL_ENDMARKER)
-			c->next.ptr = VSL_NEXT(c->next.ptr);
-		c->next.priv = c->head->seq;
-	} else
-		AZ(vslc_vsm_reset(&c->cursor));
+	i = vslc_vsm_reset(&c->cursor);
+	if (i) {
+		(void)vsl_diag(vsl, "Cursor initialization failure (%d)\n", i);
+		FREE_OBJ(c);
+		return (NULL);
+	}
 
 	return (&c->cursor);
 }



From nils.goroll at uplex.de  Fri Jun 26 12:38:38 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 26 Jun 2015 14:38:38 +0200
Subject: [master] fa4f00e Skip checks on pointers and work lists when at
 VSL_ENDMARKER
Message-ID: <E1Z8Sty-0001ZN-9h@project.varnish-software.com>


commit fa4f00e1d7654dfe12c56d1ae7122268e290773a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 24 16:33:12 2015 +0200

    Skip checks on pointers and work lists when at VSL_ENDMARKER
    
    Safe a couple of needless calls to vslc_vsm_check when we know that
    there are no new records and this no need to check shm references /
    the next cursor.

diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index b9bbcb3..52dde5f 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -125,18 +125,6 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 	assert(&c->cursor == cursor);
 	CHECK_OBJ_NOTNULL(c->vsm, VSM_MAGIC);
 
-	i = vslc_vsm_check(&c->cursor, &c->next);
-	if (i <= 0)
-		/* Overrun */
-		return (-3);
-
-	/* Check VSL fantom and abandonment */
-	if (*(volatile const uint32_t *)c->next.ptr == VSL_ENDMARKER) {
-		if (VSM_invalid == VSM_StillValid(c->vsm, &c->vf) ||
-		    VSM_Abandoned(c->vsm))
-			return (-2);
-	}
-
 	while (1) {
 		t = *(volatile const uint32_t *)c->next.ptr;
 		AN(t);
@@ -151,13 +139,18 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 		}
 
 		if (t == VSL_ENDMARKER) {
+			if (VSM_invalid == VSM_StillValid(c->vsm, &c->vf) ||
+			    VSM_Abandoned(c->vsm))
+				return (-2); /* VSL abandoned */
 			if (c->options & VSL_COPT_TAILSTOP)
-				/* EOF */
-				return (-1);
-			else
-				return (0);
+				return (-1); /* EOF */
+			return (0);	/* No new records available */
 		}
 
+		i = vslc_vsm_check(&c->cursor, &c->next);
+		if (i <= 0)
+			return (-3); /* Overrun */
+
 		c->cursor.rec = c->next;
 		c->next.ptr = VSL_NEXT(c->next.ptr);
 
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index f03f932..459d0b7 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -1320,18 +1320,18 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 	if (vslq->grouping == VSL_g_raw)
 		return (vslq_raw(vslq, func, priv));
 
+	/* Process next cursor input */
+	i = vslq_next(vslq);
+	if (i <= 0)
+		/* At end of log or cursor reports error condition */
+		return (i);
+
 	/* Check shmref list and buffer if necessary */
 	r = vslq_shmref_check(vslq);
 	if (r)
 		/* Buffering of shm ref failed */
 		return (r);
 
-	/* Process next cursor input */
-	i = vslq_next(vslq);
-	if (i < 0)
-		/* Cursor reports error condition */
-		return (i);
-
 	/* Check vtx timeout */
 	now = VTIM_mono();
 	while (!VTAILQ_EMPTY(&vslq->incomplete)) {



From nils.goroll at uplex.de  Sat Jun 27 13:14:16 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Sat, 27 Jun 2015 15:14:16 +0200
Subject: [master] a728b14 clarify how to use VSLQ_Flush()
Message-ID: <E1Z8pw0-00035U-IB@project.varnish-software.com>


commit a728b148c168637f0986b3d6745c332452384c8a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Sat Jun 27 15:14:04 2015 +0200

    clarify how to use VSLQ_Flush()

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index cebbb46..dd2f6af 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -518,12 +518,13 @@ int VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv);
 
 int VSLQ_Flush(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv);
 	/*
-	 * Flush any pending record sets from the query.
+	 * Flush any pending record sets from the query until func
+	 * (if given) returns non-zero.
 	 *
 	 * Arguments:
 	 *  vslq: The VSL context
 	 *  func: The callback function to call. Pass NULL to discard the
-	 *        pending messages
+	 *        pending messages or call repeatedly until 0 is returned.
 	 *  priv: An argument passed to func
 	 *
 	 * Return values:



From dridi.boukelmoune at gmail.com  Mon Jun 29 07:25:41 2015
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 29 Jun 2015 09:25:41 +0200
Subject: [master] ecadacb Typo
Message-ID: <E1Z9TRl-0007ib-0Q@project.varnish-software.com>


commit ecadacb3034fa0bd8b4ea4bce59fcece4c63b922
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jun 29 09:22:10 2015 +0200

    Typo

diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index 52dde5f..28e44fe 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -82,7 +82,7 @@ vslc_vsm_delete(const struct VSL_cursor *cursor)
 
 /*
  * We tolerate the fact that segment_n wraps around eventually: for the default
- * vsl_space of 80MB and 8 segments, each segement is 10MB long, so we wrap
+ * vsl_space of 80MB and 8 segments, each segment is 10MB long, so we wrap
  * roughly after 40 pebibytes (32bit) or 160 yobibytes (64bit) worth of vsl
  * written.
  *



From phk at FreeBSD.org  Mon Jun 29 08:30:13 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jun 2015 10:30:13 +0200
Subject: [master] 5177b99 Fix an assert which was (safe) off by one.
Message-ID: <E1Z9USD-0004WO-B9@project.varnish-software.com>


commit 5177b9950ca3ba6c44178cb51e8e2637e7d6d6c7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 29 08:20:21 2015 +0000

    Fix an assert which was (safe) off by one.
    
    Strengthen another assert for increased safety.
    
    Fixes: #1749

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index bdfdf0d..05edd1b 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -201,7 +201,7 @@ void
 http_SetH(const struct http *to, unsigned n, const char *fm)
 {
 
-	assert(n < to->shd);
+	assert(n < to->nhd);
 	AN(fm);
 	to->hd[n].b = TRUST_ME(fm);
 	to->hd[n].e = strchr(to->hd[n].b, '\0');
@@ -816,7 +816,7 @@ HTTP_Encode(const struct http *fm, uint8_t *p0, unsigned l, unsigned how)
 	p = p0;
 	e = p + l;
 	assert(p + 5 <= e);
-	assert(fm->nhd < fm->shd);
+	assert(fm->nhd <= fm->shd);
 	n = HTTP_HDR_FIRST - 3;
 	vbe16enc(p + 2, fm->status);
 	p += 4;



From phk at FreeBSD.org  Mon Jun 29 08:30:13 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jun 2015 10:30:13 +0200
Subject: [master] 11aaada Add (...) to silence flexelint
Message-ID: <E1Z9USD-0004WU-CJ@project.varnish-software.com>


commit 11aaada25ad9e198ed89086b455b0102e392ca67
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 29 08:21:30 2015 +0000

    Add (...) to silence flexelint

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 3566d53..06b0f49 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -456,8 +456,8 @@ VSM_Init(void)
 
 	vsl_head = VSM_Alloc(cache_param->vsl_space, VSL_CLASS, "", "");
 	AN(vsl_head);
-	vsl_segsize = (cache_param->vsl_space - sizeof *vsl_head) /
-	    sizeof *vsl_end / VSL_SEGMENTS;
+	vsl_segsize = ((cache_param->vsl_space - sizeof *vsl_head) /
+	    sizeof *vsl_end) / VSL_SEGMENTS;
 	vsl_end = vsl_head->log + vsl_segsize * VSL_SEGMENTS;
 	/* Make segment_n always overflow on first log wrap to make any
 	   problems with regard to readers on that event visible */



From phk at FreeBSD.org  Mon Jun 29 10:23:43 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jun 2015 12:23:43 +0200
Subject: [master] 6179e31 fix a race in the backend probe code
Message-ID: <E1Z9WE3-0007je-IO@project.varnish-software.com>


commit 6179e316776ba8aae31a1b5dfe7ae4d226e0e15e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 29 10:23:00 2015 +0000

    fix a race in the backend probe code
    
    Might be relevant to #1753

diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index d1613f4..489cccb 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -82,7 +82,7 @@ struct backend {
 };
 
 /*---------------------------------------------------------------------
- * Backend connection -- private to cache_backend*.c 
+ * Backend connection -- private to cache_backend*.c
  */
 
 struct vbc {
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index 2cc9673..8060bb7 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -323,6 +323,7 @@ vbp_task(struct worker *wrk, void *priv)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(vt, priv, VBP_TARGET_MAGIC);
 
+	AN(vt->running);
 	AN(vt->req);
 	assert(vt->req_len > 0);
 
@@ -332,10 +333,17 @@ vbp_task(struct worker *wrk, void *priv)
 	vbp_update_backend(vt);
 
 	Lck_Lock(&vbp_mtx);
-	if (vt->running < 0)
+	if (vt->running < 0) {
+		assert(vt->heap_idx == BINHEAP_NOIDX);
 		vbp_delete(vt);
-	else
+	} else {
 		vt->running = 0;
+		if (vt->heap_idx != BINHEAP_NOIDX) {
+			vt->due = VTIM_real() + vt->interval;
+			binheap_delete(vbp_heap, vt->heap_idx);
+			binheap_insert(vbp_heap, vt);
+		}
+	}
 	Lck_Unlock(&vbp_mtx);
 }
 
@@ -350,37 +358,31 @@ vbp_thread(struct worker *wrk, void *priv)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AZ(priv);
+	Lck_Lock(&vbp_mtx);
 	while (1) {
-		Lck_Lock(&vbp_mtx);
-		while (1) {
-			now = VTIM_real();
-			vt = binheap_root(vbp_heap);
-			if (vt == NULL) {
-				nxt = 8.192 + now;
-			} else if (vt->due > now) {
-				nxt = vt->due;
-				vt = NULL;
-			} else {
-				binheap_delete(vbp_heap, vt->heap_idx);
+		now = VTIM_real();
+		vt = binheap_root(vbp_heap);
+		if (vt == NULL) {
+			nxt = 8.192 + now;
+			(void)Lck_CondWait(&vbp_cond, &vbp_mtx, nxt);
+		} else if (vt->due > now) {
+			nxt = vt->due;
+			vt = NULL;
+			(void)Lck_CondWait(&vbp_cond, &vbp_mtx, nxt);
+		} else {
+			binheap_delete(vbp_heap, vt->heap_idx);
+			vt->due = now + vt->interval;
+			if (!vt->running) {
 				vt->running = 1;
-				vt->due = now + vt->interval;
-				binheap_insert(vbp_heap, vt);
-				nxt = 0.0;
-				break;
+				vt->task.func = vbp_task;
+				vt->task.priv = vt;
+				if (Pool_Task_Any(&vt->task, TASK_QUEUE_REQ))
+					vt->running = 0;
 			}
-			(void)Lck_CondWait(&vbp_cond, &vbp_mtx, nxt);
-		}
-		Lck_Unlock(&vbp_mtx);
-		vt->task.func = vbp_task;
-		vt->task.priv = vt;
-
-		if (Pool_Task_Any(&vt->task, TASK_QUEUE_REQ)) {
-			Lck_Lock(&vbp_mtx);
-			vt->running = 0;
-			Lck_Unlock(&vbp_mtx);
-			// XXX: ehh... ?
+			binheap_insert(vbp_heap, vt);
 		}
 	}
+	Lck_Unlock(&vbp_mtx);
 	NEEDLESS_RETURN(NULL);
 }
 
@@ -582,8 +584,10 @@ VBP_Remove(struct backend *be)
 		vt = NULL;
 	}
 	Lck_Unlock(&vbp_mtx);
-	if (vt != NULL)
+	if (vt != NULL) {
+		assert(vt->heap_idx == BINHEAP_NOIDX);
 		vbp_delete(vt);
+	}
 }
 
 /*-------------------------------------------------------------------*/
@@ -597,6 +601,9 @@ vbp_cmp(void *priv, const void *a, const void *b)
 	CAST_OBJ_NOTNULL(aa, a, VBP_TARGET_MAGIC);
 	CAST_OBJ_NOTNULL(bb, b, VBP_TARGET_MAGIC);
 
+	if (aa->running && !bb->running)
+		return (0);
+
 	return (aa->due < bb->due);
 }
 



From nils.goroll at uplex.de  Mon Jun 29 12:03:58 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 29 Jun 2015 14:03:58 +0200
Subject: [master] d571dc7 rewrite 0U - VSL_SEGMENTS in way which should make
 Flexelint happy
Message-ID: <E1Z9Xn4-0006uP-4I@project.varnish-software.com>


commit d571dc72548ce4defa00fe69e9d09b0216930a89
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jun 29 13:59:52 2015 +0200

    rewrite 0U - VSL_SEGMENTS in way which should make Flexelint happy
    
    We intentionally provoke an early wrap of our segment_n unsigned int.
    The previous code is fine according to
    http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1570.pdf
    "A computation involving unsigned operands can never overflow...",
    but nevertheless we don't want to trigger Flexelint warnings.
    
    Thanks to Dag Haavi Finstad for digging the ISO standards url.

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 06b0f49..9bb7d1e 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -461,7 +461,7 @@ VSM_Init(void)
 	vsl_end = vsl_head->log + vsl_segsize * VSL_SEGMENTS;
 	/* Make segment_n always overflow on first log wrap to make any
 	   problems with regard to readers on that event visible */
-	vsl_segment_n = 0U - VSL_SEGMENTS;
+	vsl_segment_n = UINT_MAX - VSL_SEGMENTS + 1;
 	AZ(vsl_segment_n % VSL_SEGMENTS);
 	vsl_head->segment_n = vsl_segment_n;
 	vsl_ptr = vsl_head->log;



From phk at FreeBSD.org  Mon Jun 29 12:04:40 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jun 2015 14:04:40 +0200
Subject: [master] 0d33c1d Rename this file to the name I keep typing
Message-ID: <E1Z9Xnk-00072g-Td@project.varnish-software.com>


commit 0d33c1da14232df83057d667c9a684bd88c56328
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 29 12:04:25 2015 +0000

    Rename this file to the name I keep typing

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index dc20242..dd2bae2 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -12,7 +12,7 @@ varnishd_SOURCES = \
 	cache/cache_acceptor.c \
 	cache/cache_backend.c \
 	cache/cache_backend_cfg.c \
-	cache/cache_backend_poll.c \
+	cache/cache_backend_probe.c \
 	cache/cache_backend_tcp.c \
 	cache/cache_ban.c \
 	cache/cache_busyobj.c \
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
deleted file mode 100644
index 8060bb7..0000000
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ /dev/null
@@ -1,632 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Poll backends for collection of health statistics
- *
- * We co-opt threads from the worker pool for probing the backends,
- * but we want to avoid a potentially messy cleanup operation when we
- * retire the backend, so the thread owns the health information, which
- * the backend references, rather than the other way around.
- *
- */
-
-#include "config.h"
-
-#include <poll.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "binary_heap.h"
-#include "vcli_priv.h"
-#include "vrt.h"
-#include "vsa.h"
-#include "vtcp.h"
-#include "vtim.h"
-
-#include "cache_director.h"
-#include "cache_backend.h"
-
-/* Default averaging rate, we want something pretty responsive */
-#define AVG_RATE			4
-
-struct vbp_target {
-	unsigned			magic;
-#define VBP_TARGET_MAGIC		0x6b7cb656
-
-	VRT_BACKEND_PROBE_FIELDS()
-
-	struct backend			*backend;
-	struct tcp_pool			*tcp_pool;
-
-	char				*req;
-	int				req_len;
-
-	char				resp_buf[128];
-	unsigned			good;
-
-	/* Collected statistics */
-#define BITMAP(n, c, t, b)	uint64_t	n;
-#include "tbl/backend_poll.h"
-#undef BITMAP
-
-	double				last;
-	double				avg;
-	double				rate;
-
-	double				due;
-	int				running;
-	int				heap_idx;
-	struct pool_task		task;
-};
-
-static struct lock			vbp_mtx;
-static pthread_cond_t			vbp_cond;
-static struct binheap			*vbp_heap;
-
-/*--------------------------------------------------------------------*/
-
-static void
-vbp_delete(struct vbp_target *vt)
-{
-#define DN(x)	/**/
-	VRT_BACKEND_PROBE_HANDLE();
-#undef DN
-	VBT_Rel(&vt->tcp_pool);
-	free(vt->req);
-	FREE_OBJ(vt);
-}
-
-
-/*--------------------------------------------------------------------
- * Record pokings...
- */
-
-static void
-vbp_start_poke(struct vbp_target *vt)
-{
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-
-#define BITMAP(n, c, t, b) \
-	vt->n <<= 1;
-#include "tbl/backend_poll.h"
-#undef BITMAP
-
-	vt->last = 0;
-	vt->resp_buf[0] = '\0';
-}
-
-static void
-vbp_has_poked(struct vbp_target *vt)
-{
-	unsigned i, j;
-	uint64_t u;
-
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-
-	/* Calculate exponential average */
-	if (vt->happy & 1) {
-		if (vt->rate < AVG_RATE)
-			vt->rate += 1.0;
-		vt->avg += (vt->last - vt->avg) / vt->rate;
-	}
-
-	u = vt->happy;
-	for (i = j = 0; i < vt->window; i++) {
-		if (u & 1)
-			j++;
-		u >>= 1;
-	}
-	vt->good = j;
-}
-
-static void
-vbp_update_backend(struct vbp_target *vt)
-{
-	unsigned i;
-	char bits[10];
-	const char *logmsg;
-
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-
-	Lck_Lock(&vbp_mtx);
-	if (vt->backend != NULL) {
-		i = 0;
-#define BITMAP(n, c, t, b) \
-		bits[i++] = (vt->n & 1) ? c : '-';
-#include "tbl/backend_poll.h"
-#undef BITMAP
-		bits[i] = '\0';
-
-		if (vt->good >= vt->threshold) {
-			if (vt->backend->healthy)
-				logmsg = "Still healthy";
-			else {
-				logmsg = "Back healthy";
-				vt->backend->health_changed = VTIM_real();
-			}
-			vt->backend->healthy = 1;
-		} else {
-			if (vt->backend->healthy) {
-				logmsg = "Went sick";
-				vt->backend->health_changed = VTIM_real();
-			} else
-				logmsg = "Still sick";
-			vt->backend->healthy = 0;
-		}
-		VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s",
-		    vt->backend->display_name, logmsg, bits,
-		    vt->good, vt->threshold, vt->window,
-		    vt->last, vt->avg, vt->resp_buf);
-		if (vt->backend != NULL && vt->backend->vsc != NULL)
-			vt->backend->vsc->happy = vt->happy;
-	}
-	Lck_Unlock(&vbp_mtx);
-}
-
-static void
-vbp_reset(struct vbp_target *vt)
-{
-	unsigned u;
-
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-	vt->avg = 0.0;
-	vt->rate = 0.0;
-#define BITMAP(n, c, t, b) \
-	vt->n = 0;
-#include "tbl/backend_poll.h"
-#undef BITMAP
-
-	for (u = 0; u < vt->initial; u++) {
-		vbp_start_poke(vt);
-		vt->happy |= 1;
-		vbp_has_poked(vt);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Poke one backend, once, but possibly at both IPv4 and IPv6 addresses.
- *
- * We do deliberately not use the stuff in cache_backend.c, because we
- * want to measure the backends response without local distractions.
- */
-
-static void
-vbp_poke(struct vbp_target *vt)
-{
-	int s, tmo, i;
-	double t_start, t_now, t_end;
-	unsigned rlen, resp;
-	char buf[8192], *p;
-	struct pollfd pfda[1], *pfd = pfda;
-	const struct suckaddr *sa;
-
-	t_start = t_now = VTIM_real();
-	t_end = t_start + vt->timeout;
-
-	s = VBT_Open(vt->tcp_pool, t_end - t_now, &sa);
-	if (s < 0) {
-		/* Got no connection: failed */
-		return;
-	}
-
-	i = VSA_Get_Proto(sa);
-	if (i == AF_INET)
-		vt->good_ipv4 |= 1;
-	else if(i == AF_INET6)
-		vt->good_ipv6 |= 1;
-	else
-		WRONG("Wrong probe protocol family");
-
-	t_now = VTIM_real();
-	tmo = (int)round((t_end - t_now) * 1e3);
-	if (tmo <= 0) {
-		/* Spent too long time getting it */
-		VTCP_close(&s);
-		return;
-	}
-
-	/* Send the request */
-	i = write(s, vt->req, vt->req_len);
-	if (i != vt->req_len) {
-		if (i < 0)
-			vt->err_xmit |= 1;
-		VTCP_close(&s);
-		return;
-	}
-	vt->good_xmit |= 1;
-
-	pfd->fd = s;
-	rlen = 0;
-	do {
-		pfd->events = POLLIN;
-		pfd->revents = 0;
-		tmo = (int)round((t_end - t_now) * 1e3);
-		if (tmo > 0)
-			i = poll(pfd, 1, tmo);
-		if (i == 0 || tmo <= 0) {
-			VTCP_close(&s);
-			return;
-		}
-		if (rlen < sizeof vt->resp_buf)
-			i = read(s, vt->resp_buf + rlen,
-			    sizeof vt->resp_buf - rlen);
-		else
-			i = read(s, buf, sizeof buf);
-		rlen += i;
-	} while (i > 0);
-
-	VTCP_close(&s);
-
-	if (i < 0) {
-		vt->err_recv |= 1;
-		return;
-	}
-
-	if (rlen == 0)
-		return;
-
-	/* So we have a good receive ... */
-	t_now = VTIM_real();
-	vt->last = t_now - t_start;
-	vt->good_recv |= 1;
-
-	/* Now find out if we like the response */
-	vt->resp_buf[sizeof vt->resp_buf - 1] = '\0';
-	p = strchr(vt->resp_buf, '\r');
-	if (p != NULL)
-		*p = '\0';
-	p = strchr(vt->resp_buf, '\n');
-	if (p != NULL)
-		*p = '\0';
-
-	i = sscanf(vt->resp_buf, "HTTP/%*f %u %s", &resp, buf);
-
-	if ((i == 1 || i == 2) && resp == vt->exp_status)
-		vt->happy |= 1;
-}
-
-/*--------------------------------------------------------------------
- */
-
-static void __match_proto__(task_func_t)
-vbp_task(struct worker *wrk, void *priv)
-{
-	struct vbp_target *vt;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CAST_OBJ_NOTNULL(vt, priv, VBP_TARGET_MAGIC);
-
-	AN(vt->running);
-	AN(vt->req);
-	assert(vt->req_len > 0);
-
-	vbp_start_poke(vt);
-	vbp_poke(vt);
-	vbp_has_poked(vt);
-	vbp_update_backend(vt);
-
-	Lck_Lock(&vbp_mtx);
-	if (vt->running < 0) {
-		assert(vt->heap_idx == BINHEAP_NOIDX);
-		vbp_delete(vt);
-	} else {
-		vt->running = 0;
-		if (vt->heap_idx != BINHEAP_NOIDX) {
-			vt->due = VTIM_real() + vt->interval;
-			binheap_delete(vbp_heap, vt->heap_idx);
-			binheap_insert(vbp_heap, vt);
-		}
-	}
-	Lck_Unlock(&vbp_mtx);
-}
-
-/*--------------------------------------------------------------------
- */
-
-static void * __match_proto__()
-vbp_thread(struct worker *wrk, void *priv)
-{
-	double now, nxt;
-	struct vbp_target *vt;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	AZ(priv);
-	Lck_Lock(&vbp_mtx);
-	while (1) {
-		now = VTIM_real();
-		vt = binheap_root(vbp_heap);
-		if (vt == NULL) {
-			nxt = 8.192 + now;
-			(void)Lck_CondWait(&vbp_cond, &vbp_mtx, nxt);
-		} else if (vt->due > now) {
-			nxt = vt->due;
-			vt = NULL;
-			(void)Lck_CondWait(&vbp_cond, &vbp_mtx, nxt);
-		} else {
-			binheap_delete(vbp_heap, vt->heap_idx);
-			vt->due = now + vt->interval;
-			if (!vt->running) {
-				vt->running = 1;
-				vt->task.func = vbp_task;
-				vt->task.priv = vt;
-				if (Pool_Task_Any(&vt->task, TASK_QUEUE_REQ))
-					vt->running = 0;
-			}
-			binheap_insert(vbp_heap, vt);
-		}
-	}
-	Lck_Unlock(&vbp_mtx);
-	NEEDLESS_RETURN(NULL);
-}
-
-
-/*--------------------------------------------------------------------
- * Cli functions
- */
-
-static void
-vbp_bitmap(struct cli *cli, char c, uint64_t map, const char *lbl)
-{
-	int i;
-	uint64_t u = (1ULL << 63);
-
-	VCLI_Out(cli, "  ");
-	for (i = 0; i < 64; i++) {
-		if (map & u)
-			VCLI_Out(cli, "%c", c);
-		else
-			VCLI_Out(cli, "-");
-		map <<= 1;
-	}
-	VCLI_Out(cli, " %s\n", lbl);
-}
-
-/*lint -e{506} constant value boolean */
-/*lint -e{774} constant value boolean */
-static void
-vbp_health_one(struct cli *cli, const struct vbp_target *vt)
-{
-
-	VCLI_Out(cli,
-	    "  Current states  good: %2u threshold: %2u window: %2u\n",
-	    vt->good, vt->threshold, vt->window);
-	VCLI_Out(cli,
-	    "  Average response time of good probes: %.6f\n", vt->avg);
-	VCLI_Out(cli,
-	    "  Oldest ======================"
-	    "============================ Newest\n");
-
-#define BITMAP(n, c, t, b)					\
-		if ((vt->n != 0) || (b))			\
-			vbp_bitmap(cli, (c), vt->n, (t));
-#include "tbl/backend_poll.h"
-#undef BITMAP
-}
-
-void
-VBP_Status(struct cli *cli, const struct backend *be, int details)
-{
-	struct vbp_target *vt;
-
-	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
-	vt = be->probe;
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-	VCLI_Out(cli, "%d/%d", vt->good, vt->window);
-	if (details) {
-		VCLI_Out(cli, "\n");
-		vbp_health_one(cli, vt);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Build request from probe spec
- */
-
-static void
-vbp_build_req(struct vbp_target *vt, const struct vrt_backend_probe *vbp,
-    const struct backend *be)
-{
-	struct vsb *vsb;
-
-	vsb = VSB_new_auto();
-	AN(vsb);
-	VSB_clear(vsb);
-	if(vbp->request != NULL) {
-		VSB_cat(vsb, vbp->request);
-	} else {
-		VSB_printf(vsb, "GET %s HTTP/1.1\r\n",
-		    vbp->url != NULL ?  vbp->url : "/");
-		if (be->hosthdr != NULL)
-			VSB_printf(vsb, "Host: %s\r\n", be->hosthdr);
-		VSB_printf(vsb, "Connection: close\r\n");
-		VSB_printf(vsb, "\r\n");
-	}
-	AZ(VSB_finish(vsb));
-	vt->req = strdup(VSB_data(vsb));
-	AN(vt->req);
-	vt->req_len = VSB_len(vsb);
-	VSB_delete(vsb);
-}
-
-/*--------------------------------------------------------------------
- * Sanitize and set defaults
- * XXX: we could make these defaults parameters
- */
-
-static void
-vbp_set_defaults(struct vbp_target *vt, const struct vrt_backend_probe *vp)
-{
-
-#define DN(x)	do { vt->x = vp->x; } while (0)
-	VRT_BACKEND_PROBE_HANDLE();
-#undef DN
-
-	if (vt->timeout == 0.0)
-		vt->timeout = 2.0;
-	if (vt->interval == 0.0)
-		vt->interval = 5.0;
-	if (vt->window == 0)
-		vt->window = 8;
-	if (vt->threshold == 0)
-		vt->threshold = 3;
-	if (vt->exp_status == 0)
-		vt->exp_status = 200;
-
-	if (vt->initial == ~0U)
-		vt->initial = vt->threshold - 1;
-
-	if (vt->initial > vt->threshold)
-		vt->initial = vt->threshold;
-}
-
-/*--------------------------------------------------------------------
- */
-
-void
-VBP_Control(const struct backend *be, int enable)
-{
-	struct vbp_target *vt;
-
-	ASSERT_CLI();
-	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
-	vt = be->probe;
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-
-	vbp_reset(vt);
-	vbp_update_backend(vt);
-
-	Lck_Lock(&vbp_mtx);
-	if (enable) {
-		assert(vt->heap_idx == BINHEAP_NOIDX);
-		vt->due = VTIM_real();
-		binheap_insert(vbp_heap, vt);
-		AZ(pthread_cond_signal(&vbp_cond));
-	} else {
-		assert(vt->heap_idx != BINHEAP_NOIDX);
-		binheap_delete(vbp_heap, vt->heap_idx);
-	}
-	Lck_Unlock(&vbp_mtx);
-}
-
-/*--------------------------------------------------------------------
- * Insert/Remove/Use called from cache_backend.c
- */
-
-void
-VBP_Insert(struct backend *b, const struct vrt_backend_probe *vp,
-    struct tcp_pool *tp)
-{
-	struct vbp_target *vt;
-
-	ASSERT_CLI();
-	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-	CHECK_OBJ_NOTNULL(vp, VRT_BACKEND_PROBE_MAGIC);
-
-	AZ(b->probe);
-
-	ALLOC_OBJ(vt, VBP_TARGET_MAGIC);
-	XXXAN(vt);
-
-	vt->tcp_pool = tp;
-	vt->backend = b;
-	b->probe = vt;
-
-	vbp_set_defaults(vt, vp);
-	vbp_build_req(vt, vp, b);
-
-	vbp_reset(vt);
-	vbp_update_backend(vt);
-}
-
-void
-VBP_Remove(struct backend *be)
-{
-	struct vbp_target *vt;
-
-	ASSERT_CLI();
-	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
-	vt = be->probe;
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-
-	Lck_Lock(&vbp_mtx);
-	be->healthy = 1;
-	be->probe = NULL;
-	vt->backend = NULL;
-	if (vt->running) {
-		vt->running = -1;
-		vt = NULL;
-	}
-	Lck_Unlock(&vbp_mtx);
-	if (vt != NULL) {
-		assert(vt->heap_idx == BINHEAP_NOIDX);
-		vbp_delete(vt);
-	}
-}
-
-/*-------------------------------------------------------------------*/
-
-static int __match_proto__(binheap_cmp_t)
-vbp_cmp(void *priv, const void *a, const void *b)
-{
-	const struct vbp_target *aa, *bb;
-
-	AZ(priv);
-	CAST_OBJ_NOTNULL(aa, a, VBP_TARGET_MAGIC);
-	CAST_OBJ_NOTNULL(bb, b, VBP_TARGET_MAGIC);
-
-	if (aa->running && !bb->running)
-		return (0);
-
-	return (aa->due < bb->due);
-}
-
-static void __match_proto__(binheap_update_t)
-vbp_update(void *priv, void *p, unsigned u)
-{
-	struct vbp_target *vt;
-
-	AZ(priv);
-	CAST_OBJ_NOTNULL(vt, p, VBP_TARGET_MAGIC);
-	vt->heap_idx = u;
-}
-
-/*-------------------------------------------------------------------*/
-
-void
-VBP_Init(void)
-{
-	pthread_t thr;
-
-	Lck_New(&vbp_mtx, lck_backend);
-	vbp_heap = binheap_new(NULL, vbp_cmp, vbp_update);
-	AN(vbp_heap);
-	AZ(pthread_cond_init(&vbp_cond, NULL));
-	WRK_BgThread(&thr, "Backend poller", vbp_thread, NULL);
-}
diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
new file mode 100644
index 0000000..8060bb7
--- /dev/null
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -0,0 +1,632 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Poll backends for collection of health statistics
+ *
+ * We co-opt threads from the worker pool for probing the backends,
+ * but we want to avoid a potentially messy cleanup operation when we
+ * retire the backend, so the thread owns the health information, which
+ * the backend references, rather than the other way around.
+ *
+ */
+
+#include "config.h"
+
+#include <poll.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "binary_heap.h"
+#include "vcli_priv.h"
+#include "vrt.h"
+#include "vsa.h"
+#include "vtcp.h"
+#include "vtim.h"
+
+#include "cache_director.h"
+#include "cache_backend.h"
+
+/* Default averaging rate, we want something pretty responsive */
+#define AVG_RATE			4
+
+struct vbp_target {
+	unsigned			magic;
+#define VBP_TARGET_MAGIC		0x6b7cb656
+
+	VRT_BACKEND_PROBE_FIELDS()
+
+	struct backend			*backend;
+	struct tcp_pool			*tcp_pool;
+
+	char				*req;
+	int				req_len;
+
+	char				resp_buf[128];
+	unsigned			good;
+
+	/* Collected statistics */
+#define BITMAP(n, c, t, b)	uint64_t	n;
+#include "tbl/backend_poll.h"
+#undef BITMAP
+
+	double				last;
+	double				avg;
+	double				rate;
+
+	double				due;
+	int				running;
+	int				heap_idx;
+	struct pool_task		task;
+};
+
+static struct lock			vbp_mtx;
+static pthread_cond_t			vbp_cond;
+static struct binheap			*vbp_heap;
+
+/*--------------------------------------------------------------------*/
+
+static void
+vbp_delete(struct vbp_target *vt)
+{
+#define DN(x)	/**/
+	VRT_BACKEND_PROBE_HANDLE();
+#undef DN
+	VBT_Rel(&vt->tcp_pool);
+	free(vt->req);
+	FREE_OBJ(vt);
+}
+
+
+/*--------------------------------------------------------------------
+ * Record pokings...
+ */
+
+static void
+vbp_start_poke(struct vbp_target *vt)
+{
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
+#define BITMAP(n, c, t, b) \
+	vt->n <<= 1;
+#include "tbl/backend_poll.h"
+#undef BITMAP
+
+	vt->last = 0;
+	vt->resp_buf[0] = '\0';
+}
+
+static void
+vbp_has_poked(struct vbp_target *vt)
+{
+	unsigned i, j;
+	uint64_t u;
+
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
+	/* Calculate exponential average */
+	if (vt->happy & 1) {
+		if (vt->rate < AVG_RATE)
+			vt->rate += 1.0;
+		vt->avg += (vt->last - vt->avg) / vt->rate;
+	}
+
+	u = vt->happy;
+	for (i = j = 0; i < vt->window; i++) {
+		if (u & 1)
+			j++;
+		u >>= 1;
+	}
+	vt->good = j;
+}
+
+static void
+vbp_update_backend(struct vbp_target *vt)
+{
+	unsigned i;
+	char bits[10];
+	const char *logmsg;
+
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
+	Lck_Lock(&vbp_mtx);
+	if (vt->backend != NULL) {
+		i = 0;
+#define BITMAP(n, c, t, b) \
+		bits[i++] = (vt->n & 1) ? c : '-';
+#include "tbl/backend_poll.h"
+#undef BITMAP
+		bits[i] = '\0';
+
+		if (vt->good >= vt->threshold) {
+			if (vt->backend->healthy)
+				logmsg = "Still healthy";
+			else {
+				logmsg = "Back healthy";
+				vt->backend->health_changed = VTIM_real();
+			}
+			vt->backend->healthy = 1;
+		} else {
+			if (vt->backend->healthy) {
+				logmsg = "Went sick";
+				vt->backend->health_changed = VTIM_real();
+			} else
+				logmsg = "Still sick";
+			vt->backend->healthy = 0;
+		}
+		VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s",
+		    vt->backend->display_name, logmsg, bits,
+		    vt->good, vt->threshold, vt->window,
+		    vt->last, vt->avg, vt->resp_buf);
+		if (vt->backend != NULL && vt->backend->vsc != NULL)
+			vt->backend->vsc->happy = vt->happy;
+	}
+	Lck_Unlock(&vbp_mtx);
+}
+
+static void
+vbp_reset(struct vbp_target *vt)
+{
+	unsigned u;
+
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+	vt->avg = 0.0;
+	vt->rate = 0.0;
+#define BITMAP(n, c, t, b) \
+	vt->n = 0;
+#include "tbl/backend_poll.h"
+#undef BITMAP
+
+	for (u = 0; u < vt->initial; u++) {
+		vbp_start_poke(vt);
+		vt->happy |= 1;
+		vbp_has_poked(vt);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Poke one backend, once, but possibly at both IPv4 and IPv6 addresses.
+ *
+ * We do deliberately not use the stuff in cache_backend.c, because we
+ * want to measure the backends response without local distractions.
+ */
+
+static void
+vbp_poke(struct vbp_target *vt)
+{
+	int s, tmo, i;
+	double t_start, t_now, t_end;
+	unsigned rlen, resp;
+	char buf[8192], *p;
+	struct pollfd pfda[1], *pfd = pfda;
+	const struct suckaddr *sa;
+
+	t_start = t_now = VTIM_real();
+	t_end = t_start + vt->timeout;
+
+	s = VBT_Open(vt->tcp_pool, t_end - t_now, &sa);
+	if (s < 0) {
+		/* Got no connection: failed */
+		return;
+	}
+
+	i = VSA_Get_Proto(sa);
+	if (i == AF_INET)
+		vt->good_ipv4 |= 1;
+	else if(i == AF_INET6)
+		vt->good_ipv6 |= 1;
+	else
+		WRONG("Wrong probe protocol family");
+
+	t_now = VTIM_real();
+	tmo = (int)round((t_end - t_now) * 1e3);
+	if (tmo <= 0) {
+		/* Spent too long time getting it */
+		VTCP_close(&s);
+		return;
+	}
+
+	/* Send the request */
+	i = write(s, vt->req, vt->req_len);
+	if (i != vt->req_len) {
+		if (i < 0)
+			vt->err_xmit |= 1;
+		VTCP_close(&s);
+		return;
+	}
+	vt->good_xmit |= 1;
+
+	pfd->fd = s;
+	rlen = 0;
+	do {
+		pfd->events = POLLIN;
+		pfd->revents = 0;
+		tmo = (int)round((t_end - t_now) * 1e3);
+		if (tmo > 0)
+			i = poll(pfd, 1, tmo);
+		if (i == 0 || tmo <= 0) {
+			VTCP_close(&s);
+			return;
+		}
+		if (rlen < sizeof vt->resp_buf)
+			i = read(s, vt->resp_buf + rlen,
+			    sizeof vt->resp_buf - rlen);
+		else
+			i = read(s, buf, sizeof buf);
+		rlen += i;
+	} while (i > 0);
+
+	VTCP_close(&s);
+
+	if (i < 0) {
+		vt->err_recv |= 1;
+		return;
+	}
+
+	if (rlen == 0)
+		return;
+
+	/* So we have a good receive ... */
+	t_now = VTIM_real();
+	vt->last = t_now - t_start;
+	vt->good_recv |= 1;
+
+	/* Now find out if we like the response */
+	vt->resp_buf[sizeof vt->resp_buf - 1] = '\0';
+	p = strchr(vt->resp_buf, '\r');
+	if (p != NULL)
+		*p = '\0';
+	p = strchr(vt->resp_buf, '\n');
+	if (p != NULL)
+		*p = '\0';
+
+	i = sscanf(vt->resp_buf, "HTTP/%*f %u %s", &resp, buf);
+
+	if ((i == 1 || i == 2) && resp == vt->exp_status)
+		vt->happy |= 1;
+}
+
+/*--------------------------------------------------------------------
+ */
+
+static void __match_proto__(task_func_t)
+vbp_task(struct worker *wrk, void *priv)
+{
+	struct vbp_target *vt;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(vt, priv, VBP_TARGET_MAGIC);
+
+	AN(vt->running);
+	AN(vt->req);
+	assert(vt->req_len > 0);
+
+	vbp_start_poke(vt);
+	vbp_poke(vt);
+	vbp_has_poked(vt);
+	vbp_update_backend(vt);
+
+	Lck_Lock(&vbp_mtx);
+	if (vt->running < 0) {
+		assert(vt->heap_idx == BINHEAP_NOIDX);
+		vbp_delete(vt);
+	} else {
+		vt->running = 0;
+		if (vt->heap_idx != BINHEAP_NOIDX) {
+			vt->due = VTIM_real() + vt->interval;
+			binheap_delete(vbp_heap, vt->heap_idx);
+			binheap_insert(vbp_heap, vt);
+		}
+	}
+	Lck_Unlock(&vbp_mtx);
+}
+
+/*--------------------------------------------------------------------
+ */
+
+static void * __match_proto__()
+vbp_thread(struct worker *wrk, void *priv)
+{
+	double now, nxt;
+	struct vbp_target *vt;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AZ(priv);
+	Lck_Lock(&vbp_mtx);
+	while (1) {
+		now = VTIM_real();
+		vt = binheap_root(vbp_heap);
+		if (vt == NULL) {
+			nxt = 8.192 + now;
+			(void)Lck_CondWait(&vbp_cond, &vbp_mtx, nxt);
+		} else if (vt->due > now) {
+			nxt = vt->due;
+			vt = NULL;
+			(void)Lck_CondWait(&vbp_cond, &vbp_mtx, nxt);
+		} else {
+			binheap_delete(vbp_heap, vt->heap_idx);
+			vt->due = now + vt->interval;
+			if (!vt->running) {
+				vt->running = 1;
+				vt->task.func = vbp_task;
+				vt->task.priv = vt;
+				if (Pool_Task_Any(&vt->task, TASK_QUEUE_REQ))
+					vt->running = 0;
+			}
+			binheap_insert(vbp_heap, vt);
+		}
+	}
+	Lck_Unlock(&vbp_mtx);
+	NEEDLESS_RETURN(NULL);
+}
+
+
+/*--------------------------------------------------------------------
+ * Cli functions
+ */
+
+static void
+vbp_bitmap(struct cli *cli, char c, uint64_t map, const char *lbl)
+{
+	int i;
+	uint64_t u = (1ULL << 63);
+
+	VCLI_Out(cli, "  ");
+	for (i = 0; i < 64; i++) {
+		if (map & u)
+			VCLI_Out(cli, "%c", c);
+		else
+			VCLI_Out(cli, "-");
+		map <<= 1;
+	}
+	VCLI_Out(cli, " %s\n", lbl);
+}
+
+/*lint -e{506} constant value boolean */
+/*lint -e{774} constant value boolean */
+static void
+vbp_health_one(struct cli *cli, const struct vbp_target *vt)
+{
+
+	VCLI_Out(cli,
+	    "  Current states  good: %2u threshold: %2u window: %2u\n",
+	    vt->good, vt->threshold, vt->window);
+	VCLI_Out(cli,
+	    "  Average response time of good probes: %.6f\n", vt->avg);
+	VCLI_Out(cli,
+	    "  Oldest ======================"
+	    "============================ Newest\n");
+
+#define BITMAP(n, c, t, b)					\
+		if ((vt->n != 0) || (b))			\
+			vbp_bitmap(cli, (c), vt->n, (t));
+#include "tbl/backend_poll.h"
+#undef BITMAP
+}
+
+void
+VBP_Status(struct cli *cli, const struct backend *be, int details)
+{
+	struct vbp_target *vt;
+
+	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
+	vt = be->probe;
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+	VCLI_Out(cli, "%d/%d", vt->good, vt->window);
+	if (details) {
+		VCLI_Out(cli, "\n");
+		vbp_health_one(cli, vt);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Build request from probe spec
+ */
+
+static void
+vbp_build_req(struct vbp_target *vt, const struct vrt_backend_probe *vbp,
+    const struct backend *be)
+{
+	struct vsb *vsb;
+
+	vsb = VSB_new_auto();
+	AN(vsb);
+	VSB_clear(vsb);
+	if(vbp->request != NULL) {
+		VSB_cat(vsb, vbp->request);
+	} else {
+		VSB_printf(vsb, "GET %s HTTP/1.1\r\n",
+		    vbp->url != NULL ?  vbp->url : "/");
+		if (be->hosthdr != NULL)
+			VSB_printf(vsb, "Host: %s\r\n", be->hosthdr);
+		VSB_printf(vsb, "Connection: close\r\n");
+		VSB_printf(vsb, "\r\n");
+	}
+	AZ(VSB_finish(vsb));
+	vt->req = strdup(VSB_data(vsb));
+	AN(vt->req);
+	vt->req_len = VSB_len(vsb);
+	VSB_delete(vsb);
+}
+
+/*--------------------------------------------------------------------
+ * Sanitize and set defaults
+ * XXX: we could make these defaults parameters
+ */
+
+static void
+vbp_set_defaults(struct vbp_target *vt, const struct vrt_backend_probe *vp)
+{
+
+#define DN(x)	do { vt->x = vp->x; } while (0)
+	VRT_BACKEND_PROBE_HANDLE();
+#undef DN
+
+	if (vt->timeout == 0.0)
+		vt->timeout = 2.0;
+	if (vt->interval == 0.0)
+		vt->interval = 5.0;
+	if (vt->window == 0)
+		vt->window = 8;
+	if (vt->threshold == 0)
+		vt->threshold = 3;
+	if (vt->exp_status == 0)
+		vt->exp_status = 200;
+
+	if (vt->initial == ~0U)
+		vt->initial = vt->threshold - 1;
+
+	if (vt->initial > vt->threshold)
+		vt->initial = vt->threshold;
+}
+
+/*--------------------------------------------------------------------
+ */
+
+void
+VBP_Control(const struct backend *be, int enable)
+{
+	struct vbp_target *vt;
+
+	ASSERT_CLI();
+	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
+	vt = be->probe;
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
+	vbp_reset(vt);
+	vbp_update_backend(vt);
+
+	Lck_Lock(&vbp_mtx);
+	if (enable) {
+		assert(vt->heap_idx == BINHEAP_NOIDX);
+		vt->due = VTIM_real();
+		binheap_insert(vbp_heap, vt);
+		AZ(pthread_cond_signal(&vbp_cond));
+	} else {
+		assert(vt->heap_idx != BINHEAP_NOIDX);
+		binheap_delete(vbp_heap, vt->heap_idx);
+	}
+	Lck_Unlock(&vbp_mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Insert/Remove/Use called from cache_backend.c
+ */
+
+void
+VBP_Insert(struct backend *b, const struct vrt_backend_probe *vp,
+    struct tcp_pool *tp)
+{
+	struct vbp_target *vt;
+
+	ASSERT_CLI();
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+	CHECK_OBJ_NOTNULL(vp, VRT_BACKEND_PROBE_MAGIC);
+
+	AZ(b->probe);
+
+	ALLOC_OBJ(vt, VBP_TARGET_MAGIC);
+	XXXAN(vt);
+
+	vt->tcp_pool = tp;
+	vt->backend = b;
+	b->probe = vt;
+
+	vbp_set_defaults(vt, vp);
+	vbp_build_req(vt, vp, b);
+
+	vbp_reset(vt);
+	vbp_update_backend(vt);
+}
+
+void
+VBP_Remove(struct backend *be)
+{
+	struct vbp_target *vt;
+
+	ASSERT_CLI();
+	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
+	vt = be->probe;
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
+	Lck_Lock(&vbp_mtx);
+	be->healthy = 1;
+	be->probe = NULL;
+	vt->backend = NULL;
+	if (vt->running) {
+		vt->running = -1;
+		vt = NULL;
+	}
+	Lck_Unlock(&vbp_mtx);
+	if (vt != NULL) {
+		assert(vt->heap_idx == BINHEAP_NOIDX);
+		vbp_delete(vt);
+	}
+}
+
+/*-------------------------------------------------------------------*/
+
+static int __match_proto__(binheap_cmp_t)
+vbp_cmp(void *priv, const void *a, const void *b)
+{
+	const struct vbp_target *aa, *bb;
+
+	AZ(priv);
+	CAST_OBJ_NOTNULL(aa, a, VBP_TARGET_MAGIC);
+	CAST_OBJ_NOTNULL(bb, b, VBP_TARGET_MAGIC);
+
+	if (aa->running && !bb->running)
+		return (0);
+
+	return (aa->due < bb->due);
+}
+
+static void __match_proto__(binheap_update_t)
+vbp_update(void *priv, void *p, unsigned u)
+{
+	struct vbp_target *vt;
+
+	AZ(priv);
+	CAST_OBJ_NOTNULL(vt, p, VBP_TARGET_MAGIC);
+	vt->heap_idx = u;
+}
+
+/*-------------------------------------------------------------------*/
+
+void
+VBP_Init(void)
+{
+	pthread_t thr;
+
+	Lck_New(&vbp_mtx, lck_backend);
+	vbp_heap = binheap_new(NULL, vbp_cmp, vbp_update);
+	AN(vbp_heap);
+	AZ(pthread_cond_init(&vbp_cond, NULL));
+	WRK_BgThread(&thr, "Backend poller", vbp_thread, NULL);
+}



From martin at varnish-software.com  Mon Jun 29 15:00:15 2015
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 29 Jun 2015 17:00:15 +0200
Subject: [master] 7fdf6d0 Fix out of storage condition when failing to
 allocate transient synth
Message-ID: <E1Z9aXf-0003oI-Io@project.varnish-software.com>


commit 7fdf6d09fbf30c53f51e9da085fb1a3a8d1d5cc5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 29 16:38:08 2015 +0200

    Fix out of storage condition when failing to allocate transient synth
    
    Do not assert if the transient stevedore fails to provide storage for
    the synth body.
    
    Log an SLT_Error record when failing on out of storage.
    
    Document that synth objects are created on transient storage.
    
    Fix an objcore leak that would come if we failed to create the object.
    
    Fixes: #1740

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9c4861f..b0fe1e7 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -223,6 +223,8 @@ cnt_synth(struct worker *wrk, struct req *req)
 	struct http *h;
 	double now;
 	struct vsb *synth_body;
+	ssize_t sz, szl;
+	uint8_t *ptr;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -269,22 +271,25 @@ cnt_synth(struct worker *wrk, struct req *req)
 	req->objcore = HSH_Private(wrk);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 	if (STV_NewObject(req->objcore, wrk, TRANSIENT_STORAGE, 1024)) {
-		ssize_t sz, szl;
-		uint8_t *ptr;
-
 		szl = VSB_len(synth_body);
 		assert(szl >= 0);
-		if (szl > 0) {
-			sz = szl;
-			AN(ObjGetSpace(wrk, req->objcore, &sz, &ptr));
-			assert(sz >= szl);
+	} else
+		szl = -1;
+	if (szl > 0) {
+		sz = szl;
+		if (ObjGetSpace(wrk, req->objcore, &sz, &ptr) && sz >= szl) {
 			memcpy(ptr, VSB_data(synth_body), szl);
 			ObjExtend(wrk, req->objcore, szl);
-		}
-
-		cnt_vdp(req, NULL);
-		(void)HSH_DerefObjCore(wrk, &req->objcore);
+		} else
+			szl = -1;
 	}
+	if (szl < 0) {
+		VSLb(req->vsl, SLT_Error, "Could not get storage");
+		req->doclose = SC_OVERLOAD;
+	} else
+		cnt_vdp(req, NULL);
+
+	(void)HSH_DerefObjCore(wrk, &req->objcore);
 	VSB_delete(synth_body);
 
 	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst
index 5d08d1e..9782313 100644
--- a/doc/sphinx/users-guide/storage-backends.rst
+++ b/doc/sphinx/users-guide/storage-backends.rst
@@ -132,8 +132,9 @@ previously banned objects to reappear.
 Transient Storage
 -----------------
 
-If you name any of your storage backend "Transient" it will be
-used for transient (short lived) objects. By default Varnish
+If you name any of your storage backend "Transient" it will be used
+for transient (short lived) objects. This includes the temporary
+objects created when returning a synthetic object. By default Varnish
 would use an unlimited malloc backend for this.
 
 .. XXX: Is this another paramater? In that case handled in the same manner as above? benc



From phk at FreeBSD.org  Tue Jun 30 05:38:05 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 30 Jun 2015 07:38:05 +0200
Subject: [master] 7585d89 Sort things tbl/vcl_returns.h
Message-ID: <E1Z9oFB-00027R-BV@project.varnish-software.com>


commit 7585d89ac28eb9be95d6c9510b8f77ecae1b8c2e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 29 20:21:46 2015 +0000

    Sort things tbl/vcl_returns.h
    
    Various white-space fixes

diff --git a/CONTRIBUTING b/CONTRIBUTING
index 94a668f..3f15742 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -15,4 +15,3 @@ Patches can be sent to varnish-dev at varnish-cache.org.
 
 Documentation fixes and other small items for the documentation team may be
 sent as Github pull requests.
-
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 489cccb..cc27de6 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -132,4 +132,3 @@ void VBT_Close(struct tcp_pool *tp, struct vbc **vbc);
 struct vbc *VBT_Get(struct tcp_pool *, double tmo, const struct backend *,
     struct worker *);
 void VBT_Wait(struct worker *, struct vbc *);
-
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 2124c33..699bcf5 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -1098,7 +1098,9 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 		if (i) {
 			VSLb(vsl, SLT_ExpBan, "%u banned by lurker",
 			    ObjGetXID(wrk, oc));
-			EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0); // XXX fake now
+
+			EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0);
+					// XXX ^ fake now
 			VSC_C_main->bans_lurker_obj_killed++;
 		}
 		(void)HSH_DerefObjCore(wrk, &oc);
diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h
index 66addc5..f717b78 100644
--- a/bin/varnishd/http1/cache_http1.h
+++ b/bin/varnishd/http1/cache_http1.h
@@ -61,4 +61,3 @@ void V1L_Reserve(struct worker *, struct ws *, int *fd, struct vsl_log *,
 unsigned V1L_Flush(const struct worker *w);
 unsigned V1L_FlushRelease(struct worker *w);
 size_t V1L_Write(const struct worker *w, const void *ptr, ssize_t len);
-
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 86fcdee..2b680c6 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -151,7 +151,8 @@ void STV_Config(const char *spec);
 void STV_Config_Transient(void);
 
 /* mgt_vcc.c */
-char *mgt_VccCompile(struct cli *cli, const char *vclname, const char *vclsrc, int C_flag);
+char *mgt_VccCompile(struct cli *, const char *vclname, const char *vclsrc,
+    int C_flag);
 void mgt_vcc_init(void);
 
 void mgt_vcl_init(void);
diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c
index f0c0443..1f0962d 100644
--- a/bin/varnishd/mgt/mgt_sandbox.c
+++ b/bin/varnishd/mgt/mgt_sandbox.c
@@ -233,7 +233,8 @@ mgt_sandbox_unix(enum sandbox_e who)
 
 #ifdef __linux__
 	/*
-	 * On linux mucking about with uid/gid disables core-dumps,			 * reenable them again.
+	 * On linux mucking about with uid/gid disables core-dumps,
+	 * reenable them again.
 	 */
 	if (prctl(PR_SET_DUMPABLE, 1) != 0) {
 		REPORT0(LOG_INFO,
diff --git a/include/vlu.h b/include/vlu.h
index b001523..1b5cd4a 100644
--- a/include/vlu.h
+++ b/include/vlu.h
@@ -14,7 +14,7 @@
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
diff --git a/include/vsub.h b/include/vsub.h
index 274663a..bc337a1 100644
--- a/include/vsub.h
+++ b/include/vsub.h
@@ -31,5 +31,5 @@
 /* from libvarnish/subproc.c */
 typedef void vsub_func_f(void*);
 
-unsigned VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name,
+unsigned VSUB_run(struct vsb *, vsub_func_f *, void *priv, const char *name,
     int maxlines);
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 39810fc..6d6915a 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -976,24 +976,28 @@ fo.write("\n/*lint -save -e525 -e539 */\n")
 fo.write("\n#ifdef VCL_RET_MAC\n")
 l = list(rets.keys())
 l.sort()
+ll = list(returns)
+ll.sort()
 for i in l:
 	fo.write("VCL_RET_MAC(%s, %s" % (i.lower(), i.upper()))
-	s=", "
-	for j in returns:
+	s=",\n\t"
+	for j in ll:
 		if i in j[2]:
 			fo.write("%sVCL_MET_%s" % (s, j[0].upper()))
-			s = " | "
-	fo.write(")\n")
+			s = " |\n\t"
+	fo.write("\n)\n")
 fo.write("#endif\n")
 
 fo.write("\n#ifdef VCL_MET_MAC\n")
-for i in returns:
-	fo.write("VCL_MET_MAC(%s,%s,\n" % (i[0].lower(), i[0].upper()))
-	p = " ("
-	for j in i[2]:
-		fo.write("    %s(1U << VCL_RET_%s)\n" % (p, j.upper()))
-		p = "| "
-	fo.write("))\n")
+for i in ll:
+	fo.write("VCL_MET_MAC(%s, %s," % (i[0].lower(), i[0].upper()))
+	p = " (\n\t"
+	lll = list(i[2])
+	lll.sort()
+	for j in lll:
+		fo.write("%s(1U << VCL_RET_%s)" % (p, j.upper()))
+		p = " |\n\t"
+	fo.write("\n))\n")
 fo.write("#endif\n")
 fo.write("\n/*lint -restore */\n")
 fo.close()
@@ -1035,7 +1039,9 @@ def tbl40(a, b):
 fo.write("\n/* VCL Methods */\n")
 n = 1
 for i in returns:
-	fo.write(tbl40("#define VCL_MET_%s" % i[0].upper(),  "(1U << %d)\n" % n))
+	fo.write(
+	    tbl40("#define VCL_MET_%s" % i[0].upper(), "(1U << %d)\n" % n)
+	)
 	n += 1
 
 fo.write("\n" + tbl40("#define VCL_MET_MAX", "%d\n" % n))
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index df5aa43..b9c6531 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -79,7 +79,8 @@ vcc_TimeUnit(struct vcc *tl)
 	else {
 		VSB_printf(tl->sb, "Unknown time unit ");
 		vcc_ErrToken(tl, tl->t);
-		VSB_printf(tl->sb, ".  Legal are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'\n");
+		VSB_printf(tl->sb,
+		    ".  Legal are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'\n");
 		vcc_ErrWhere(tl, tl->t);
 		return (1.0);
 	}
diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index ccfdce1..2e958a7 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -178,5 +178,5 @@ Description
 
 	Use the string or list of strings provided to pick the backend.
 Example
-	set req.backend_hint = vdir.backend(req.http.cookie);  # pick a backend based on the cookie header from the client
-
+	# pick a backend based on the cookie header from the client
+	set req.backend_hint = vdir.backend(req.http.cookie);
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 097cd45..e8478e8 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -247,4 +247,3 @@ vmod_strstr(VRT_CTX, VCL_STRING s1, VCL_STRING s2)
 		return (NULL);
 	return (strstr(s1, s2));
 }
-



From phk at FreeBSD.org  Tue Jun 30 21:50:20 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 30 Jun 2015 23:50:20 +0200
Subject: [master] b90a835 Lock the backend tcp-pools under the backends_mtx,
 rather than the (implicit) CLI lock.
Message-ID: <E1ZA3Q4-0005zN-Op@project.varnish-software.com>


commit b90a835497d04154948370fadfcc3d1b58e85970
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 30 21:49:40 2015 +0000

    Lock the backend tcp-pools under the backends_mtx, rather than the
    (implicit) CLI lock.

diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 7291cd4..057dea4 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -49,6 +49,7 @@
 
 static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends);
 static struct lock backends_mtx;
+
 /*--------------------------------------------------------------------
  * Create/Delete a new director::backend instance.
  */
@@ -59,6 +60,7 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 	struct backend *b;
 	char buf[128];
 	struct vcl *vcl;
+	struct tcp_pool *tp = NULL;
 	const struct vrt_backend_probe *vbp;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
@@ -87,25 +89,28 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 
 	b->vcl = vcl;
 
-	b->tcp_pool = VBT_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr);
-
 	b->healthy = 1;
 	b->health_changed = VTIM_real();
 	b->admin_health = ah_probe;
 
+	vbp = vrt->probe;
+	if (vbp == NULL)
+		vbp = VCL_DefaultProbe(vcl);
+
 	Lck_Lock(&backends_mtx);
 	VTAILQ_INSERT_TAIL(&backends, b, list);
 	VSC_C_main->n_backend++;
+	b->tcp_pool = VBT_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr);
+	if (vbp != NULL) {
+		tp = VBT_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr);
+		assert(b->tcp_pool == tp);
+	}
 	Lck_Unlock(&backends_mtx);
 
 	VBE_fill_director(b);
 
-	vbp = vrt->probe;
-	if (vbp == NULL)
-		vbp = VCL_DefaultProbe(vcl);
 	if (vbp != NULL)
-		VBP_Insert(b, vbp,
-		    VBT_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr));
+		VBP_Insert(b, vbp, tp);
 
 	VCL_AddBackend(ctx->vcl, b);
 
@@ -167,6 +172,7 @@ VBE_Delete(struct backend *be)
 	Lck_Lock(&backends_mtx);
 	VTAILQ_REMOVE(&backends, be, list);
 	VSC_C_main->n_backend--;
+	VBT_Rel(&be->tcp_pool);
 	Lck_Unlock(&backends_mtx);
 
 #define DA(x)	do { if (be->x != NULL) free(be->x); } while (0)
@@ -177,7 +183,6 @@ VBE_Delete(struct backend *be)
 
 	free(be->display_name);
 	AZ(be->vsc);
-	VBT_Rel(&be->tcp_pool);
 	Lck_Delete(&be->mtx);
 	FREE_OBJ(be);
 }
diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index 05fd232..3e99387 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -127,7 +127,6 @@ VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6)
 {
 	struct tcp_pool *tp;
 
-	ASSERT_CLI();
 	VTAILQ_FOREACH(tp, &pools, list) {
 		assert(tp->refcnt > 0);
 		if (ip4 == NULL) {