From phk at FreeBSD.org  Mon May  4 09:05:09 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 04 May 2015 11:05:09 +0200
Subject: [master] 9ea6149 Add a tiny bit of compile time type-safety,
 just because we can.
Message-ID: <E1YpCJJ-00033C-Cz@project.varnish-software.com>


commit 9ea6149ae1d37aa4053449a490c9b4938e38aba3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 4 09:04:50 2015 +0000

    Add a tiny bit of compile time type-safety, just because we can.

diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c
index f3e2519..5245708 100644
--- a/bin/varnishd/cache/cache_vrt_vmod.c
+++ b/bin/varnishd/cache/cache_vrt_vmod.c
@@ -62,7 +62,7 @@ struct vmod {
 static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
 
 int
-VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
+VRT_Vmod_Init(struct vmod **hdl, void *ptr, int len, const char *nm,
     const char *path, const char *file_id, VRT_CTX)
 {
 	struct vmod *v;
@@ -73,6 +73,8 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 	ASSERT_CLI();
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(ctx->cli);
+	AN(hdl);
+	AZ(*hdl);
 
 	dlhdl = dlopen(path, RTLD_NOW | RTLD_LOCAL);
 	if (dlhdl == NULL) {
@@ -141,15 +143,16 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 }
 
 void
-VRT_Vmod_Fini(void **hdl)
+VRT_Vmod_Fini(struct vmod **hdl)
 {
 	struct vmod *v;
 
 	ASSERT_CLI();
 
-	AN(*hdl);
-	CAST_OBJ_NOTNULL(v, *hdl, VMOD_MAGIC);
+	AN(hdl);
+	v = *hdl;
 	*hdl = NULL;
+	CHECK_OBJ_NOTNULL(v, VMOD_MAGIC);
 
 #ifndef DONT_DLCLOSE_VMODS
 	/*
diff --git a/include/vrt.h b/include/vrt.h
index e034f91..5e4882e 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -58,6 +58,7 @@ struct cli;
 struct director;
 struct VCL_conf;
 struct suckaddr;
+struct vmod;
 
 /***********************************************************************
  * This is the central definition of the mapping from VCL types to
@@ -243,9 +244,9 @@ void VRT_fini_vbe(VRT_CTX, struct director **, const struct vrt_backend *);
 int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst);
 
 /* VMOD/Modules related */
-int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
+int VRT_Vmod_Init(struct vmod **hdl, void *ptr, int len, const char *nm,
     const char *path, const char *file_id, VRT_CTX);
-void VRT_Vmod_Fini(void **hdl);
+void VRT_Vmod_Fini(struct vmod **hdl);
 
 struct vmod_priv;
 typedef void vmod_priv_free_f(void *);
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 9564ca9..8391c98 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -218,7 +218,7 @@ vcc_ParseImport(struct vcc *tl)
 	}
 
 	Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod));
-	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
+	Fh(tl, 0, "static struct vmod *VGC_vmod_%.*s;\n", PF(mod));
 	Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod));
 	Fh(tl, 0, "\n%s\n", vmd->proto);
 	Fh(tl, 0, "\n/* --- END VMOD %.*s --- */\n\n", PF(mod));



From tfheen at err.no  Mon May  4 11:28:51 2015
From: tfheen at err.no (Tollef Fog Heen)
Date: Mon, 04 May 2015 13:28:51 +0200
Subject: [master] 419f983 Enable PCRE JIT-er by default
Message-ID: <E1YpEYN-0002R2-KY@project.varnish-software.com>


commit 419f983eeec6d0b0b932105495f4e09179260eec
Author: Tollef Fog Heen <tfheen at err.no>
Date:   Mon May 4 13:27:17 2015 +0200

    Enable PCRE JIT-er by default
    
    The JIT-er is generally safe to use, and faster, so use that.
    
    Fixes: #1576, #1630

diff --git a/configure.ac b/configure.ac
index fafe5e4..6c4d5f2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -614,9 +614,9 @@ AC_DEFINE_UNQUOTED([VCC_CC],"$VCC_CC",[C compiler command line for VCL code])
 # --enable-pcre-jit
 AC_ARG_ENABLE(pcre-jit,
     AS_HELP_STRING([--enable-pcre-jit],
-	[use the PCRE JIT compiler (default is NO)]),
+	[use the PCRE JIT compiler (default is YES)]),
     ,
-    [enable_pcre_jit=no])
+    [enable_pcre_jit=yes])
 
 if test "$enable_pcre_jit" = yes; then
    AC_DEFINE([USE_PCRE_JIT],[1],[use the PCRE JIT compiler])



From phk at FreeBSD.org  Mon May  4 18:55:40 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 04 May 2015 20:55:40 +0200
Subject: [master] e012bc3 Introduce a req->resp_len field to hold the number
 of bytes we actually expect to deliver,
 and use it to eliminate most of the C-L header munging.
Message-ID: <E1YpLWm-00010T-S7@project.varnish-software.com>


commit e012bc3e5489785339b678945bd7ef23e04df1ea
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 4 18:54:51 2015 +0000

    Introduce a req->resp_len field to hold the number of bytes we actually
    expect to deliver, and use it to eliminate most of the C-L header munging.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b9c1dd7..54422bc 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -591,7 +591,10 @@ struct req {
 	/* HTTP request */
 	struct http		*http;
 	struct http		*http0;
+
+	/* HTTP response */
 	struct http		*resp;
+	intmax_t		resp_len;
 
 	struct ws		ws[1];
 	struct objcore		*objcore;
diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c
index a74b41f..86f4df2 100644
--- a/bin/varnishd/cache/cache_deliver_proc.c
+++ b/bin/varnishd/cache/cache_deliver_proc.c
@@ -140,7 +140,7 @@ VDP_DeliverObj(struct req *req)
 			WRONG("Wrong OIS value");
 		}
 	} while (ois == OIS_DATA || ois == OIS_STREAM);
+	(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
 	ObjIterEnd(req->objcore, &oi);
 	return (ois);
 }
-
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index a5c79a2..d8edb91 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -635,6 +635,7 @@ ESI_DeliverChild(struct req *req, struct busyobj *bo)
 		if (bo != NULL)
 			VBO_waitstate(bo, BOS_FINISHED);
 		ved_stripgzip(req);
+		(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
 	} else {
 		if (req->gzip_resp && !i)
 			VDP_push(req, ved_pretend_gzip, NULL, 0);
@@ -643,6 +644,5 @@ ESI_DeliverChild(struct req *req, struct busyobj *bo)
 
 		(void)VDP_DeliverObj(req);
 	}
-	(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
 	VDP_close(req);
 }
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 652af2f..041d890 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -293,14 +293,14 @@ VDP_gunzip(struct req *req, enum vdp_action act, void **priv,
 		VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
 		*priv = vg;
 
-		http_Unset(req->resp, H_Content_Length);
 		p = ObjGetattr(req->wrk, req->objcore, OA_GZIPBITS, &dl);
 		if (p != NULL && dl == 32) {
 			u = vbe64dec(p + 24);
 			/* XXX: Zero is suspect: OA_GZIPBITS wasn't set */
 			if (u != 0)
-				http_PrintfHeader(req->resp,
-				    "Content-Length: %ju", (uintmax_t)u);
+				req->resp_len = u;
+			else
+				req->resp_len = -1;
 		}
 		http_Unset(req->resp, H_Content_Encoding);
 		return (0);
diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index be29ece..1cb50fa 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -151,9 +151,7 @@ VRG_dorange(struct req *req, struct busyobj *bo, const char *r)
 
 	http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd",
 	    (intmax_t)low, (intmax_t)high, (intmax_t)len);
-	http_Unset(req->resp, H_Content_Length);
-	http_PrintfHeader(req->resp, "Content-Length: %jd",
-	    (intmax_t)(1 + high - low));
+	req->resp_len = (intmax_t)(1 + high - low);
 	http_PutResponse(req->resp, "HTTP/1.1", 206, NULL);
 
 	vrg_priv = WS_Alloc(req->ws, sizeof *vrg_priv);
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index 5db8754..27d4f72 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -70,6 +70,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
 	req->res_mode = 0;
+	req->resp_len = -2;
 
 	/*
 	 * Determine ESI status first.  Not dependent on wantbody, because
@@ -92,12 +93,8 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 	} else if (http_IsStatus(req->resp, 304)) {
 		http_Unset(req->resp, H_Content_Length);
 		req->wantbody = 0;
-	} else if (bo == NULL &&
-	    !http_GetHdr(req->resp, H_Content_Length, NULL)) {
-		http_PrintfHeader(req->resp,
-		    "Content-Length: %ju", (uintmax_t)ObjGetLen(
-		    req->wrk, req->objcore));
-	}
+	} else if (bo == NULL && req->wantbody)
+		req->resp_len = ObjGetLen(req->wrk, req->objcore);
 
 	if (cache_param->http_gzip_support &&
 	    ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
@@ -114,6 +111,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 	if (req->res_mode & RES_ESI) {
 		/* Gunzip could have added back a C-L */
 		http_Unset(req->resp, H_Content_Length);
+		req->resp_len = -1;
 	}
 
 	/*
@@ -126,11 +124,15 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 			VRG_dorange(req, bo, r);
 	}
 
+	if (req->resp_len >= -1 && req->wantbody)
+		http_Unset(req->resp, H_Content_Length);
+	if (req->resp_len >= 0 && req->wantbody)
+		http_PrintfHeader(req->resp,
+		    "Content-Length: %jd", req->resp_len);
 
 	if (http_GetHdr(req->resp, H_Content_Length, NULL))
 		req->res_mode |= RES_LEN;
-
-	if (req->wantbody && !(req->res_mode & RES_LEN)) {
+	else if (req->wantbody) {
 		if (req->http->protover == 11) {
 			req->res_mode |= RES_CHUNKED;
 			http_SetHeader(req->resp, "Transfer-Encoding: chunked");
@@ -162,10 +164,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 	if (req->wantbody) {
 		if (req->res_mode & RES_CHUNKED)
 			V1L_Chunked(req->wrk);
-
 		ois = VDP_DeliverObj(req);
-		(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
-
 		if (ois == OIS_DONE && (req->res_mode & RES_CHUNKED))
 			V1L_EndChunk(req->wrk);
 	}



From phk at FreeBSD.org  Mon May  4 19:50:01 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 04 May 2015 21:50:01 +0200
Subject: [master] 6acf177 Major tour through the Range code to make us more
 strict and RFC7233 conforming.
Message-ID: <E1YpMNN-0006YK-H9@project.varnish-software.com>


commit 6acf17779787ab832d4c2acf51d8941892e3e140
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 4 19:49:29 2015 +0000

    Major tour through the Range code to make us more strict and RFC7233 conforming.

diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index 1cb50fa..7562095 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -82,72 +82,64 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv,
 
 /*--------------------------------------------------------------------*/
 
-void
-VRG_dorange(struct req *req, struct busyobj *bo, const char *r)
+static int
+vrg_dorange(struct req *req, const struct busyobj *bo, ssize_t len,
+    const char *r)
 {
-	ssize_t len, low, high, has_low;
+	ssize_t low, high, has_low, has_high, t;
 	struct vrg_priv *vrg_priv;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	assert(http_IsStatus(req->resp, 200));
-
-	/* We must snapshot the length if we're streaming from the backend */
-	if (bo != NULL)
-		len = VBO_waitlen(req->wrk, bo, -1);
-	else
-		len = ObjGetLen(req->wrk, req->objcore);
-
+	(void)bo;
 	if (strncasecmp(r, "bytes=", 6))
-		return;
+		return (__LINE__);
 	r += 6;
 
 	/* The low end of range */
 	has_low = low = 0;
 	while (vct_isdigit(*r)) {
 		has_low = 1;
+		t = low;
 		low *= 10;
-		low += *r - '0';
-		r++;
+		low += *r++ - '0';
+		if (low < t)
+			return (__LINE__);
 	}
 
-	if (*r != '-')
-		return;
-	r++;
+	if (*r++ != '-')
+		return (__LINE__);
 
 	/* The high end of range */
-	if (vct_isdigit(*r)) {
-		high = 0;
-		while (vct_isdigit(*r)) {
-			high *= 10;
-			high += *r - '0';
-			r++;
-		}
-		if (high < low)
-			return;
-		if (!has_low) {
-			low = len - high;
-			if (low < 0)
-				low = 0;
-			high = len - 1;
-		} else if (high >= len)
-			high = len - 1;
-	} else if (has_low)
-		high = len - 1;
-	else
-		return;
+	has_high = high = 0;
+	while (vct_isdigit(*r)) {
+		has_high = 1;
+		t = high;
+		high *= 10;
+		high += *r++ - '0';
+		if (high < t)
+			return (__LINE__);
+	}
 
 	if (*r != '\0')
-		return;
+		return (__LINE__);
 
-	if (low >= len) {
-		http_PrintfHeader(req->resp, "Content-Range: bytes */%jd",
-		    (intmax_t)len);
-		http_Unset(req->resp, H_Content_Length);
-		http_PutResponse(req->resp, "HTTP/1.1", 416, NULL);
-		req->wantbody = 0;
-		return;
-	}
+	if (has_high + has_low == 0)
+		return (__LINE__);
+
+	if (!has_low) {
+		if (high == 0)
+			return (__LINE__);
+		low = len - high;
+		if (low < 0)
+			low = 0;
+		high = len - 1;
+	} else if (high >= len || !has_high)
+		high = len - 1;
+
+	if (high < low)
+		return (__LINE__);
+
+	if (low >= len)
+		return (__LINE__);
 
 	http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd",
 	    (intmax_t)low, (intmax_t)high, (intmax_t)len);
@@ -161,4 +153,33 @@ VRG_dorange(struct req *req, struct busyobj *bo, const char *r)
 	vrg_priv->range_low = low;
 	vrg_priv->range_high = high + 1;
 	VDP_push(req, vrg_range_bytes, vrg_priv, 1);
+	return (0);
+}
+
+void
+VRG_dorange(struct req *req, struct busyobj *bo, const char *r)
+{
+	size_t len;
+	int i;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+	assert(http_IsStatus(req->resp, 200));
+
+	/* We must snapshot the length if we're streaming from the backend */
+	if (bo != NULL)
+		len = VBO_waitlen(req->wrk, bo, -1);
+	else
+		len = ObjGetLen(req->wrk, req->objcore);
+
+	i = vrg_dorange(req, bo, len, r);
+	if (i) {
+		VSLb(req->vsl, SLT_Debug, "RANGE_FAIL line %d", i);
+		http_Unset(req->resp, H_Content_Length);
+		if (bo == NULL)
+			http_PrintfHeader(req->resp,
+			    "Content-Range: bytes */%jd", (intmax_t)len);
+		http_PutResponse(req->resp, "HTTP/1.1", 416, NULL);
+		req->wantbody = 0;
+	}
 }
diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index 9eeee15..f479498 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -5,12 +5,14 @@ server s1 {
 	txresp -bodylen 100
 } -start
 
-varnish v1 -arg "-p http_range_support=off" -vcl+backend {
+varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
 	}
 } -start
 
+varnish v1 -cliok "param.set http_range_support off"
+
 client c1 {
 	txreq -hdr "Range: bytes=0-9"
 	rxresp
@@ -18,61 +20,61 @@ client c1 {
 	expect resp.bodylen == 100
 } -run
 
-varnish v1 -cliok "param.set http_range_support on"
-
 varnish v1 -expect s_resp_bodybytes == 100
 
+varnish v1 -cliok "param.set http_range_support on"
+
 client c1 {
+	# Invalid range requests
+
 	txreq -hdr "Range: bytes =0-9"
 	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 100
+	expect resp.status == 416
+	expect resp.bodylen == 0
+	expect resp.http.content-range == "bytes */100"
 
 	txreq -hdr "Range: bytes=0- 9"
 	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 100
+	expect resp.status == 416
+	expect resp.bodylen == 0
 
 	txreq -hdr "Range: bytes =-9"
 	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 100
+	expect resp.status == 416
+	expect resp.bodylen == 0
 
 	txreq -hdr "Range: bytes =0-a"
 	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 100
+	expect resp.status == 416
+	expect resp.bodylen == 0
 
 	txreq -hdr "Range: bytes=-"
 	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 100
+	expect resp.status == 416
+	expect resp.bodylen == 0
 
 	txreq -hdr "Range: bytes=5-2"
 	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 100
-} -run
-
-varnish v1 -expect s_resp_bodybytes == 700
+	expect resp.status == 416
+	expect resp.bodylen == 0
 
-client c1 {
 	txreq -hdr "Range: bytes=-0"
 	rxresp
 	expect resp.status == 416
 	expect resp.bodylen == 0
-	expect resp.http.content-range == "bytes */100"
 
 	txreq -hdr "Range: bytes=100-"
 	rxresp
 	expect resp.status == 416
 	expect resp.bodylen == 0
-	expect resp.http.content-range == "bytes */100"
 } -run
 
-varnish v1 -expect s_resp_bodybytes == 700
+varnish v1 -expect s_resp_bodybytes == 100
+delay .1
 
 client c1 {
+	# Valid range requests
+
 	txreq -hdr "Range: bytes=0-49"
 	rxresp
 	expect resp.status == 206
@@ -110,4 +112,4 @@ client c1 {
 	expect resp.http.content-range == "bytes 0-99/100"
 } -run
 
-varnish v1 -expect s_resp_bodybytes == 1001
+varnish v1 -expect s_resp_bodybytes == 401
diff --git a/bin/varnishtest/tests/r00466.vtc b/bin/varnishtest/tests/r00466.vtc
index b672681..813c1e7 100644
--- a/bin/varnishtest/tests/r00466.vtc
+++ b/bin/varnishtest/tests/r00466.vtc
@@ -28,11 +28,13 @@ varnish v1 -vcl+backend {
 varnish v1 -cliok "param.set debug +syncvsl"
 
 client c1 {
-	txreq -url "/foo" -hdr "Range: 100-200"
+	txreq -url "/foo" -hdr "Range: bytes=1-2"
 	rxresp
-	expect resp.status == 200
+	expect resp.status == 206
+	expect resp.http.Content-Length == 2
 	expect resp.http.X-Varnish == "1001"
 
+	# NB: Deliberately bogus Range header
 	txreq -url "/bar" -hdr "Range: 200-300"
 	rxresp
 	expect resp.status == 206
diff --git a/bin/varnishtest/tests/r01660.vtc b/bin/varnishtest/tests/r01660.vtc
index 1ce133c..5e2599d 100644
--- a/bin/varnishtest/tests/r01660.vtc
+++ b/bin/varnishtest/tests/r01660.vtc
@@ -14,5 +14,10 @@ varnish v1 -vcl+backend {
 client c1 {
 	txreq -hdr "Range: 0-1"
 	rxresp
-	expect resp.status == 200
+	expect resp.status == 416
+
+	txreq -hdr "Range: bytes=0-1"
+	rxresp
+	expect resp.status == 206
+	expect resp.http.content-length == 2
 } -run



From fgsch at lodoss.net  Mon May  4 20:20:12 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 04 May 2015 22:20:12 +0200
Subject: [master] 4e5c8ab Fail gracefully if we can't get a backend on pipe
Message-ID: <E1YpMqa-0001vc-SU@project.varnish-software.com>


commit 4e5c8ab06ee081aca68dc04ea91493778ae2a040
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon May 4 20:39:23 2015 +0100

    Fail gracefully if we can't get a backend on pipe
    
    Fixes #1730

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 114ab38..c555e05 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -277,6 +277,10 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	i = vbe_dir_getfd(d, bo);
+	if (i < 0) {
+		VSLb(bo->vsl, SLT_FetchError, "no backend connection");
+		return;
+	}
 	V1P_Process(req, bo, i);
 	vbe_dir_finish(d, bo->wrk, bo);
 }
diff --git a/bin/varnishtest/tests/r01730.vtc b/bin/varnishtest/tests/r01730.vtc
new file mode 100644
index 0000000..d19b479
--- /dev/null
+++ b/bin/varnishtest/tests/r01730.vtc
@@ -0,0 +1,15 @@
+varnishtest "Test connection error on pipe"
+
+varnish v1 -vcl {
+       backend default { .host = "${bad_ip}"; }
+       sub vcl_recv {
+               return (pipe);
+       }
+} -start
+
+client c1 {
+       txreq
+       expect_close
+} -run
+
+varnish v1 -expect sc_rx_timeout == 1



From fgsch at lodoss.net  Mon May  4 20:20:12 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 04 May 2015 22:20:12 +0200
Subject: [master] a8d0651 One "no backend connection" is enough
Message-ID: <E1YpMqa-0001vW-RI@project.varnish-software.com>


commit a8d065135b9e63735abe83b88e8011f8f8658e60
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon May 4 20:31:46 2015 +0100

    One "no backend connection" is enough
    
    Fixes #1724

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index a02a7d8..114ab38 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -110,7 +110,6 @@ vbe_dir_getfd(const struct director *d, struct busyobj *bo)
 	if (vc == NULL) {
 		// XXX: Per backend stats ?
 		VSC_C_main->backend_fail++;
-		VSLb(bo->vsl, SLT_FetchError, "no backend connection");
 		return (-1);
 	}
 



From fgsch at lodoss.net  Mon May  4 20:20:12 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 04 May 2015 22:20:12 +0200
Subject: [master] dd54ea0 Testcase for #1576
Message-ID: <E1YpMqa-0001vl-Tm@project.varnish-software.com>


commit dd54ea0e4318f1ef9fa40f128b8b599cf4f40bfc
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon May 4 21:08:06 2015 +0100

    Testcase for #1576

diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc
new file mode 100644
index 0000000..67f5041
--- /dev/null
+++ b/bin/varnishtest/tests/r01576.vtc
@@ -0,0 +1,31 @@
+varnishtest "Test certain regex fail before consuming all the stack"
+
+server s1 {
+	rxreq
+	expect req.http.found == "1"
+	txresp
+	rxreq
+	expect req.http.found == <undef>
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.url ~ "^/a((?!/.).)*$") {
+			set req.http.found = "1";
+		}
+	}
+} -start
+
+logexpect l1 -v v1 {
+	expect * * VCL_Error "Regexp matching returned -27"
+} -start
+
+client c1 {
+	txreq -url /aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+	rxresp
+	txreq -url /aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+	rxresp
+} -run
+
+logexpect l1 -wait



From phk at FreeBSD.org  Mon May  4 21:15:55 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 04 May 2015 23:15:55 +0200
Subject: [master] 24f88f8 Add a session close value for clients asking for
 bogus ranges on streaming objects.
Message-ID: <E1YpNiV-0000c0-79@project.varnish-software.com>


commit 24f88f89f8dcb3394085bff2344e4da64d5681ab
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 4 21:05:52 2015 +0000

    Add a session close value for clients asking for bogus ranges
    on streaming objects.

diff --git a/include/tbl/sess_close.h b/include/tbl/sess_close.h
index 21c78ab..ef4c946 100644
--- a/include/tbl/sess_close.h
+++ b/include/tbl/sess_close.h
@@ -44,5 +44,6 @@ SESS_CLOSE(TX_EOF,	  tx_eof,	0,	"EOF transmission")
 SESS_CLOSE(RESP_CLOSE,	  resp_close,	0,	"Backend/VCL requested close")
 SESS_CLOSE(OVERLOAD,	  overload,	1,	"Out of some resource")
 SESS_CLOSE(PIPE_OVERFLOW, pipe_overflow,1,	"Session pipe overflow")
+SESS_CLOSE(RANGE_SHORT,   range_short,  1,	"Insufficient data for range")
 
 /*lint -restore */



From phk at FreeBSD.org  Mon May  4 21:15:55 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 04 May 2015 23:15:55 +0200
Subject: [master] cfd288a Add "recv N" action
Message-ID: <E1YpNiV-0000bu-5v@project.varnish-software.com>


commit cfd288a5dfd6720fc586f91cf6cf176be21873ed
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 4 20:36:36 2015 +0000

    Add "recv N" action

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 229663d..663ba49 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -976,6 +976,34 @@ cmd_http_txreq(CMD_ARGS)
 }
 
 /**********************************************************************
+ * Receive N characters
+ */
+
+static void
+cmd_http_recv(CMD_ARGS)
+{
+	struct http *hp;
+	int i, n;
+	uint8_t u[32];
+
+	(void)cmd;
+	(void)vl;
+	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
+	AN(av[1]);
+	AZ(av[2]);
+	n = strtoul(av[1], NULL, 0);
+	while (n > 0) {
+		i = read(hp->fd, u, n > 32 ? 32 : n);
+		if (i > 0)
+			vtc_dump(hp->vl, 4, "recv", u, i);
+		else 
+			vtc_log(hp->vl, hp->fatal, "recv() got %d (%s)", i,
+			    strerror(errno));
+		n -= i;
+	}
+}
+
+/**********************************************************************
  * Send a string
  */
 
@@ -1307,6 +1335,7 @@ static const struct cmds http_cmds[] = {
 	{ "rxrespbody",		cmd_http_rxrespbody },
 	{ "gunzip",		cmd_http_gunzip_body },
 	{ "expect",		cmd_http_expect },
+	{ "recv",		cmd_http_recv },
 	{ "send",		cmd_http_send },
 	{ "send_n",		cmd_http_send_n },
 	{ "send_urgent",	cmd_http_send_urgent },



From phk at FreeBSD.org  Mon May  4 21:15:55 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 04 May 2015 23:15:55 +0200
Subject: [master] cbb0356 Communicate the session close reason as a negative
 in the sess->fd so the SessClose VSL can reporte it correctly.
Message-ID: <E1YpNiV-0000c9-8U@project.varnish-software.com>


commit cbb035694ddb802e3e2a8db5b2f6b1f0c62a4459
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 4 21:12:23 2015 +0000

    Communicate the session close reason as a negative in the sess->fd
    so the SessClose VSL can reporte it correctly.
    
    Probably broken by: Martin ?

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 6391cd7..820c349 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -530,7 +530,7 @@ SES_Close(struct sess *sp, enum sess_close reason)
 	assert(sp->fd >= 0);
 	i = close(sp->fd);
 	assert(i == 0 || errno != EBADF); /* XXX EINVAL seen */
-	sp->fd = -1;
+	sp->fd = -(int)reason;
 	if (reason != SC_NULL)
 		ses_close_acct(reason);
 }
@@ -557,6 +557,9 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
 		now = VTIM_real();
 	AZ(isnan(sp->t_open));
 
+	if (reason == SC_NULL)
+		reason = (enum sess_close)-sp->fd;
+
 	assert(VTAILQ_EMPTY(&sp->privs->privs));
 	VSL(SLT_SessClose, sp->vxid, "%s %.3f",
 	    sess_close_2str(reason, 0), now - sp->t_open);



From phk at FreeBSD.org  Mon May  4 21:15:55 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 04 May 2015 23:15:55 +0200
Subject: [master] 97434c7 Handle Range requests on streaming objects better.
Message-ID: <E1YpNiV-0000cI-9y@project.varnish-software.com>


commit 97434c7cef70d99533f1fb7f78419dd5c9d36d23
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 4 21:13:36 2015 +0000

    Handle Range requests on streaming objects better.
    
    If the client sends [LO]-HI range, we trust it knows something we don't (yet),
    and do the Range request.  If we later ran out of data, we close the session.
    
    Other range requests on streaming objects are ignored.
    
    Fixes: #1506
    Fixes: #1618

diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index 7562095..31ec81e 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -56,11 +56,14 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv,
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	if (act == VDP_INIT)
 		return (0);
+	CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC);
 	if (act == VDP_FINI) {
-		*priv = NULL;
+		if (vrg_priv->range_off < vrg_priv->range_high)
+			SES_Close(req->sp, SC_RANGE_SHORT);
+		*priv = NULL;	/* struct on ws, no need to free */
 		return (0);
 	}
-	CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC);
+
 	l = vrg_priv->range_low - vrg_priv->range_off;
 	if (l > 0) {
 		if (l > len)
@@ -126,19 +129,29 @@ vrg_dorange(struct req *req, const struct busyobj *bo, ssize_t len,
 		return (__LINE__);
 
 	if (!has_low) {
+		if (bo != NULL)
+			return (0);		// Allow 200 response
 		if (high == 0)
 			return (__LINE__);
 		low = len - high;
 		if (low < 0)
 			low = 0;
 		high = len - 1;
-	} else if (high >= len || !has_high)
+	} else if (bo == NULL && (high >= len || !has_high))
 		high = len - 1;
+	else if (!has_high)
+		return (0);			// Allow 200 response
+	/*
+	 * else (bo != NULL) {
+	 *    We assume that the client knows what it's doing and trust
+	 *    that both low and high make sense.
+	 * }
+	 */
 
 	if (high < low)
 		return (__LINE__);
 
-	if (low >= len)
+	if (bo == NULL && low >= len)
 		return (__LINE__);
 
 	http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd",
diff --git a/bin/varnishtest/tests/r01506.vtc b/bin/varnishtest/tests/r01506.vtc
index 09a8db0..f2bb5b2 100644
--- a/bin/varnishtest/tests/r01506.vtc
+++ b/bin/varnishtest/tests/r01506.vtc
@@ -1,45 +1,72 @@
-varnishtest "Streaming of C-L header from backend"
+varnishtest "range requests on streamed response"
 
-server s1 {
+server s1 -repeat 4 {
 	rxreq
-	txresp -nolen -hdr "Content-Length: 020"
-	send "0123456789"
+	txresp -nolen \
+	    -hdr "Transfer-Encoding: chunked" \
+	    -hdr "Connection: close"
+	send "11\r\n0_23456789abcdef\n"
+	send "11\r\n1_23456789abcdef\n"
+	send "11\r\n2_23456789abcdef\n"
+	send "11\r\n3_23456789abcdef\n"
 	sema r1 sync 2
-	send "0123456789"
+	send "11\r\n4_23456789abcdef\n"
+	send "11\r\n5_23456789abcdef\n"
+	send "11\r\n6_23456789abcdef\n"
+	send "11\r\n7_23456789abcdef\n"
+	chunkedlen 0
 
-	rxreq
-	txresp -nolen -hdr "Content-Length: 010"
-	sema r2 sync 2
-	send "0123456789"
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -vcl+backend {} -start
+
+varnish v1 -cliok "param.set debug +syncvsl"
 
-	sub vcl_backend_response {
-		if (bereq.url == "/2") {
-			set beresp.do_gzip = true;
-		}
-	}
-} -start -cliok "param.set debug +syncvsl" -cliok "param.set debug +flush_head"
+logexpect l1 -v v1 -g session {
+	expect 0 1000	Begin		sess
+	expect * =	SessClose	RANGE_SHORT
+} -start
 
 client c1 {
-	txreq
+	txreq -url /1 -hdr "Range: bytes=17-101"
+	rxresphdrs
+	expect resp.status == 206
+	expect resp.http.content-length == 85
+	sema r1 sync 2
+	rxrespbody
+	expect resp.bodylen == 85
+	delay .1
+
+	# We cannot do tail-ranges when streaming
+	txreq -url /2 -hdr "Range: bytes=-10"
 	rxresphdrs
-	expect resp.http.content-length == "020"
+	expect resp.status == 200
+	expect resp.http.Transfer-Encoding == chunked
 	sema r1 sync 2
 	rxrespbody
-	expect resp.body == "01234567890123456789"
+	expect resp.bodylen == 136
 	delay .1
 
-	txreq -url "/2"
+	# We cannot do open-ranges when streaming
+	txreq -url /3 -hdr "Range: bytes=17-"
 	rxresphdrs
-	expect resp.http.content-length == "<undef>"
-	sema r2 sync 2
+	expect resp.status == 200
+	expect resp.http.Transfer-Encoding == chunked
+	sema r1 sync 2
 	rxrespbody
-	expect resp.body == "0123456789"
+	expect resp.bodylen == 136
 	delay .1
 
-	txreq -url "/2"
-	rxresp
-	expect resp.http.content-length == "10"
+	# Invalid range
+	txreq -url /4 -hdr "Range: bytes=102-200"
+	rxresphdrs
+	expect resp.status == 206
+	expect resp.http.content-length == 99
+	sema r1 sync 2
+	recv 34
+	delay .3
+	expect_close
 } -run
+
+varnish v1 -expect sc_range_short == 1
+logexpect l1 -wait



From phk at FreeBSD.org  Mon May  4 21:31:27 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 04 May 2015 23:31:27 +0200
Subject: [master] 8552e75 Use consistent type for buffers.
Message-ID: <E1YpNxX-0002sl-6O@project.varnish-software.com>


commit 8552e75ccb144ed9d7710ff1e0fb0c9e552119db
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 4 21:31:06 2015 +0000

    Use consistent type for buffers.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 663ba49..0e4cfbf 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -984,7 +984,7 @@ cmd_http_recv(CMD_ARGS)
 {
 	struct http *hp;
 	int i, n;
-	uint8_t u[32];
+	char u[32];
 
 	(void)cmd;
 	(void)vl;



From phk at FreeBSD.org  Tue May  5 07:33:17 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 09:33:17 +0200
Subject: [master] ffe29fe Workaround: Compile even if PCRE is too old to have
 JIT
Message-ID: <E1YpXLx-0006Dq-Jn@project.varnish-software.com>


commit ffe29fee279a576f978230c7169bb72ac4c86b91
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 07:32:47 2015 +0000

    Workaround:  Compile even if PCRE is too old to have JIT
    
    The real fix should be in configure

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index cc5a615..193afa9 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -36,7 +36,7 @@
 
 #include "vre.h"
 
-#if USE_PCRE_JIT
+#if defined(USE_PCRE_JIT) && defined (PCRE_STUDY_JIT_COMPILE)
 #define VRE_STUDY_JIT_COMPILE PCRE_STUDY_JIT_COMPILE
 #else
 #define VRE_STUDY_JIT_COMPILE 0



From phk at FreeBSD.org  Tue May  5 07:51:35 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 09:51:35 +0200
Subject: [master] ce89ba7 Only enable PCRE-JIT for PCRE version >= 8.32
Message-ID: <E1YpXdf-0008C7-P6@project.varnish-software.com>


commit ce89ba7fbdbeb684789db7c2834bdbff4856ea7f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 07:51:15 2015 +0000

    Only enable PCRE-JIT for PCRE version >= 8.32

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 193afa9..94400fa 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -36,7 +36,7 @@
 
 #include "vre.h"
 
-#if defined(USE_PCRE_JIT) && defined (PCRE_STUDY_JIT_COMPILE)
+#if defined(USE_PCRE_JIT) && PCRE_MAJOR == 8 && PCRE_MINOR >= 32
 #define VRE_STUDY_JIT_COMPILE PCRE_STUDY_JIT_COMPILE
 #else
 #define VRE_STUDY_JIT_COMPILE 0



From phk at FreeBSD.org  Tue May  5 08:12:39 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 10:12:39 +0200
Subject: [master] bb99710 Let varnishtest know if we use pcre_jit
Message-ID: <E1YpXy3-0002Jd-DQ@project.varnish-software.com>


commit bb997101a0eb9793348768c76aa5bdc3a345b259
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 08:12:23 2015 +0000

    Let varnishtest know if we use pcre_jit

diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc
index 67f5041..5978504 100644
--- a/bin/varnishtest/tests/r01576.vtc
+++ b/bin/varnishtest/tests/r01576.vtc
@@ -1,5 +1,7 @@
 varnishtest "Test certain regex fail before consuming all the stack"
 
+feature pcre_jit
+
 server s1 {
 	rxreq
 	expect req.http.found == "1"
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 653045f..4b40c90 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -47,6 +47,7 @@
 
 #include "vav.h"
 #include "vnum.h"
+#include "vre.h"
 #include "vtim.h"
 
 #define		MAX_TOKENS		200
@@ -598,6 +599,9 @@ cmd_feature(CMD_ARGS)
 		    getgrnam("varnish") != NULL)
 			continue;
 
+		if (!strcmp(av[i], "pcre_jit") && vre__jit)
+			continue;
+
 		vtc_log(vl, 1, "SKIPPING test, missing feature: %s", av[i]);
 		vtc_stop = 1;
 		return;
diff --git a/include/vre.h b/include/vre.h
index a59e8d7..301e806 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -51,6 +51,8 @@ typedef struct vre vre_t;
 extern const unsigned VRE_CASELESS;
 extern const unsigned VRE_NOTEMPTY;
 
+extern const int vre__jit;
+
 vre_t *VRE_compile(const char *, int, const char **, int *);
 int VRE_exec(const vre_t *code, const char *subject, int length,
     int startoffset, int options, int *ovector, int ovecsize,
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 94400fa..0115bad 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -46,6 +46,8 @@
 #  define pcre_free_study pcre_free
 #endif
 
+const int vre__jit = VRE_STUDY_JIT_COMPILE;
+
 struct vre {
 	unsigned		magic;
 #define VRE_MAGIC		0xe83097dc



From phk at FreeBSD.org  Tue May  5 08:59:13 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 10:59:13 +0200
Subject: [master] f6d7149 Add an assert so I don't again botch the SessClose
 duration field.
Message-ID: <E1YpYh7-000752-Ba@project.varnish-software.com>


commit f6d71499cccfb75ac4d709d66841ac80628239c0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 08:58:39 2015 +0000

    Add an assert so I don't again botch the SessClose duration field.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 820c349..1f6a8ef 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -560,6 +560,7 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
 	if (reason == SC_NULL)
 		reason = (enum sess_close)-sp->fd;
 
+	assert(now > sp->t_open);
 	assert(VTAILQ_EMPTY(&sp->privs->privs));
 	VSL(SLT_SessClose, sp->vxid, "%s %.3f",
 	    sess_close_2str(reason, 0), now - sp->t_open);
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index acded52..9d08d29 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -201,16 +201,16 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 				Req_Release(req);
 				switch(hs) {
 				case HTC_S_CLOSE:
-					SES_Delete(sp, SC_REM_CLOSE, 0.0);
+					SES_Delete(sp, SC_REM_CLOSE, NAN);
 					return;
 				case HTC_S_TIMEOUT:
-					SES_Delete(sp, SC_RX_TIMEOUT, 0.0);
+					SES_Delete(sp, SC_RX_TIMEOUT, NAN);
 					return;
 				case HTC_S_OVERFLOW:
-					SES_Delete(sp, SC_RX_OVERFLOW, 0.0);
+					SES_Delete(sp, SC_RX_OVERFLOW, NAN);
 					return;
 				case HTC_S_EOF:
-					SES_Delete(sp, SC_REM_CLOSE, 0.0);
+					SES_Delete(sp, SC_REM_CLOSE, NAN);
 					return;
 				default:
 					WRONG("htc_status (bad)");



From phk at FreeBSD.org  Tue May  5 09:50:58 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 11:50:58 +0200
Subject: [master] b43f31e Give chunked precedence over C-L per RFC7230
Message-ID: <E1YpZVC-0005Md-9P@project.varnish-software.com>


commit b43f31ec9ce1f32e007ba79f3c292721e916b0a2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 09:49:38 2015 +0000

    Give chunked precedence over C-L per RFC7230

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 0e4cfbf..4e40ae1 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -472,15 +472,6 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 
 	hp->body = hp->rxbuf + hp->prxbuf;
 	ll = 0;
-	p = http_find_header(hh, "content-length");
-	if (p != NULL) {
-		l = strtoul(p, NULL, 10);
-		(void)http_rxchar(hp, l, 0);
-		vtc_dump(hp->vl, 4, "body", hp->body, l);
-		hp->bodyl = l;
-		sprintf(hp->bodylen, "%d", l);
-		return;
-	}
 	p = http_find_header(hh, "transfer-encoding");
 	if (p != NULL && !strcasecmp(p, "chunked")) {
 		while (http_rxchunk(hp) != 0)
@@ -491,6 +482,15 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 		sprintf(hp->bodylen, "%d", ll);
 		return;
 	}
+	p = http_find_header(hh, "content-length");
+	if (p != NULL) {
+		l = strtoul(p, NULL, 10);
+		(void)http_rxchar(hp, l, 0);
+		vtc_dump(hp->vl, 4, "body", hp->body, l);
+		hp->bodyl = l;
+		sprintf(hp->bodylen, "%d", l);
+		return;
+	}
 	if (body) {
 		do  {
 			i = http_rxchar(hp, 1, 1);



From phk at FreeBSD.org  Tue May  5 09:50:58 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 11:50:58 +0200
Subject: [master] 9ab5669 Remove any C-L header if T-E: chunked is present.
Message-ID: <E1YpZVC-0005Mj-AY@project.varnish-software.com>


commit 9ab5669b8684add053650ff724cfe75cecfa324b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 09:50:03 2015 +0000

    Remove any C-L header if T-E: chunked is present.
    
    Fixes: #1729

diff --git a/bin/varnishd/http1/cache_http1_proto.c b/bin/varnishd/http1/cache_http1_proto.c
index 29110fb..fc20bae 100644
--- a/bin/varnishd/http1/cache_http1_proto.c
+++ b/bin/varnishd/http1/cache_http1_proto.c
@@ -263,7 +263,7 @@ http1_splitline(struct http *hp, const struct http_conn *htc, const int *hf)
 /*--------------------------------------------------------------------*/
 
 static enum body_status
-http1_body_status(const struct http *hp, struct http_conn *htc)
+http1_body_status(struct http *hp, struct http_conn *htc)
 {
 	ssize_t cl;
 	const char *b;
@@ -273,8 +273,10 @@ http1_body_status(const struct http *hp, struct http_conn *htc)
 
 	htc->content_length = -1;
 
-	if (http_HdrIs(hp, H_Transfer_Encoding, "chunked"))
+	if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) {
+		http_Unset(hp, H_Content_Length);
 		return (BS_CHUNKED);
+	}
 
 	if (http_GetHdr(hp, H_Transfer_Encoding, &b))
 		return (BS_ERROR);
diff --git a/bin/varnishtest/tests/r01729.vtc b/bin/varnishtest/tests/r01729.vtc
new file mode 100644
index 0000000..dd7e0da
--- /dev/null
+++ b/bin/varnishtest/tests/r01729.vtc
@@ -0,0 +1,75 @@
+varnishtest "C-L/T-E:chunked conflict"
+
+server s1 {
+	rxreqhdrs
+	expect req.http.content-length == <undef>
+	expect req.http.transfer-encoding == chunked
+	rxreqbody
+	expect req.url == /1
+	expect req.bodylen == 20
+
+	send "HTTP/1.1 200 OK"
+	send "Content-Length: 31\r\n"
+	send "Transfer-Encoding: chunked\r\n"
+	send "\r\n"
+	send "14\r\n"
+	send "0123456789"
+	send "0123456789"
+	send "0\r\n"
+	send "\r\n"
+
+	rxreqhdrs
+	expect req.http.content-length == 20
+	expect req.http.transfer-encoding == <undef>
+	expect req.url == /2
+	expect req.bodylen == 20
+
+	send "HTTP/1.1 200 OK"
+	send "Content-Length: 31\r\n"
+	send "Transfer-Encoding: chunked\r\n"
+	send "\r\n"
+	send "14\r\n"
+	send "0123456789"
+	send "0123456789"
+	send "0\r\n"
+	send "\r\n"
+
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_std};
+	sub vcl_recv {
+		if (req.url == "/2") {
+			std.cache_req_body(1KB);
+		}
+	}
+} -start
+
+client c1 {
+
+	send "PUT /1 HTTP/1.1\r\n"
+	send "Content-Length: 31\r\n"
+	send "Transfer-Encoding: chunked\r\n"
+	send "\r\n"
+	send "14\r\n"
+	send "0123456789"
+	send "0123456789"
+	send "0\r\n"
+	send "\r\n"
+	rxresp
+	expect resp.bodylen == 20
+
+
+	send "PUT /2 HTTP/1.1\r\n"
+	send "Content-Length: 31\r\n"
+	send "Transfer-Encoding: chunked\r\n"
+	send "\r\n"
+	send "14\r\n"
+	send "0123456789"
+	send "0123456789"
+	send "0\r\n"
+	send "\r\n"
+	rxresp
+	expect resp.bodylen == 20
+
+} -run



From phk at FreeBSD.org  Tue May  5 09:50:58 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 11:50:58 +0200
Subject: [master] 18a06da Add a test for HTTP/0.9 deprecation
Message-ID: <E1YpZVC-0005Ms-Bq@project.varnish-software.com>


commit 18a06da35bf736a88314f469d24711794d31e04f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 09:50:45 2015 +0000

    Add a test for HTTP/0.9 deprecation

diff --git a/bin/varnishtest/tests/b00047.vtc b/bin/varnishtest/tests/b00047.vtc
new file mode 100644
index 0000000..081cb0e
--- /dev/null
+++ b/bin/varnishtest/tests/b00047.vtc
@@ -0,0 +1,33 @@
+varnishtest "Check that all but HTTP/1.0 and HTTP/1.1 get 400"
+
+server s1 {
+	rxreq
+	txresp -body "FOO"
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	send "GET /\r\n\r\n"
+	rxresp
+	expect resp.status == 400
+} -run
+
+client c1 {
+	send "GET / HTTP/0.5\r\n\r\n"
+	rxresp
+	expect resp.status == 400
+} -run
+
+client c1 {
+	send "GET / HTTP/1.2\r\n\r\n"
+	rxresp
+	expect resp.status == 400
+} -run
+
+client c1 {
+	send "GET / HTTP/2.0\r\n\r\n"
+	rxresp
+	expect resp.status == 400
+} -run



From martin at varnish-software.com  Tue May  5 10:57:39 2015
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 05 May 2015 12:57:39 +0200
Subject: [master] 93820a1 Implement a resp.is_streaming status variable
Message-ID: <E1YpaXj-0002zE-S7@project.varnish-software.com>


commit 93820a16c0d50c4e0b00e4d570eb4478873b4918
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 5 11:49:32 2015 +0200

    Implement a resp.is_streaming status variable
    
    This variable will be true when the response object's body will be
    streamed from the backend.

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index e900726..5ae624e 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -658,6 +658,19 @@ VRT_r_obj_uncacheable(VRT_CTX)
 
 /*--------------------------------------------------------------------*/
 
+unsigned
+VRT_r_resp_is_streaming(VRT_CTX)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	if (ctx->req->objcore == NULL)
+		return (0);	/* When called from vcl_synth */
+	CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC);
+	return (ctx->req->objcore->busyobj != NULL ? 1 : 0);
+}
+
+/*--------------------------------------------------------------------*/
+
 #define HTTP_VAR(x)						\
 struct http *							\
 VRT_r_##x(VRT_CTX)				\
diff --git a/bin/varnishtest/tests/c00069.vtc b/bin/varnishtest/tests/c00069.vtc
new file mode 100644
index 0000000..fc41422
--- /dev/null
+++ b/bin/varnishtest/tests/c00069.vtc
@@ -0,0 +1,39 @@
+varnishtest "Test resp.is_streaming"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Content-Length: 10"
+	delay 1
+	send "1234567890"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.url == "/synth") {
+			return(synth(200, "OK"));
+		}
+	}
+	sub vcl_synth {
+		set resp.http.streaming = resp.is_streaming;
+	}
+	sub vcl_deliver {
+		set resp.http.streaming = resp.is_streaming;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.streaming == "true"
+
+	delay 0.1
+
+	txreq
+	rxresp
+	expect resp.http.streaming == "false"
+
+	txreq -url /synth
+	rxresp
+	expect resp.http.streaming == "false"
+} -run
+
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 2c1226f..c541e8f 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -670,6 +670,14 @@ sp_variables = [
 		The corresponding HTTP header.
 		"""
 	),
+        ('resp.is_streaming',
+                'BOOL',
+                ( 'deliver', 'synth', ),
+                ( ), """
+                Returns true when the response will be streamed
+                from the backend.
+                """
+        ),
 	('now',
 		'TIME',
 		( 'all',),



From phk at FreeBSD.org  Tue May  5 12:29:24 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 14:29:24 +0200
Subject: [master] ea7a162 If there is a C-L in the object,
 trust it for Range header purposes.
Message-ID: <E1YpbyW-0003rL-84@project.varnish-software.com>


commit ea7a162f81bb5fd6a39d4ca6cea3d052e15e9dc0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 12:29:01 2015 +0000

    If there is a C-L in the object, trust it for Range header purposes.

diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index 31ec81e..3bb4753 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -86,13 +86,11 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv,
 /*--------------------------------------------------------------------*/
 
 static int
-vrg_dorange(struct req *req, const struct busyobj *bo, ssize_t len,
-    const char *r)
+vrg_dorange(struct req *req, ssize_t len, const char *r)
 {
 	ssize_t low, high, has_low, has_high, t;
 	struct vrg_priv *vrg_priv;
 
-	(void)bo;
 	if (strncasecmp(r, "bytes=", 6))
 		return (__LINE__);
 	r += 6;
@@ -129,7 +127,7 @@ vrg_dorange(struct req *req, const struct busyobj *bo, ssize_t len,
 		return (__LINE__);
 
 	if (!has_low) {
-		if (bo != NULL)
+		if (len < 0)
 			return (0);		// Allow 200 response
 		if (high == 0)
 			return (__LINE__);
@@ -137,7 +135,7 @@ vrg_dorange(struct req *req, const struct busyobj *bo, ssize_t len,
 		if (low < 0)
 			low = 0;
 		high = len - 1;
-	} else if (bo == NULL && (high >= len || !has_high))
+	} else if (len >= 0 && (high >= len || !has_high))
 		high = len - 1;
 	else if (!has_high)
 		return (0);			// Allow 200 response
@@ -151,11 +149,15 @@ vrg_dorange(struct req *req, const struct busyobj *bo, ssize_t len,
 	if (high < low)
 		return (__LINE__);
 
-	if (bo == NULL && low >= len)
+	if (len >= 0 && low >= len)
 		return (__LINE__);
 
-	http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd",
-	    (intmax_t)low, (intmax_t)high, (intmax_t)len);
+	if (len >= 0)
+		http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd",
+		    (intmax_t)low, (intmax_t)high, (intmax_t)len);
+	else
+		http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/*",
+		    (intmax_t)low, (intmax_t)high);
 	req->resp_len = (intmax_t)(1 + high - low);
 	http_PutResponse(req->resp, "HTTP/1.1", 206, NULL);
 
@@ -172,7 +174,7 @@ vrg_dorange(struct req *req, const struct busyobj *bo, ssize_t len,
 void
 VRG_dorange(struct req *req, struct busyobj *bo, const char *r)
 {
-	size_t len;
+	ssize_t len;
 	int i;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -180,12 +182,17 @@ VRG_dorange(struct req *req, struct busyobj *bo, const char *r)
 	assert(http_IsStatus(req->resp, 200));
 
 	/* We must snapshot the length if we're streaming from the backend */
-	if (bo != NULL)
-		len = VBO_waitlen(req->wrk, bo, -1);
-	else
+	if (bo != NULL) {
+		len = http_GetContentLength(bo->beresp);
+VSLb(req->vsl, SLT_Debug, "UUU 1 %jd", (intmax_t)len);
+#if 0
+		if (len < 0)
+			len = VBO_waitlen(req->wrk, bo, -1);
+#endif
+	} else
 		len = ObjGetLen(req->wrk, req->objcore);
 
-	i = vrg_dorange(req, bo, len, r);
+	i = vrg_dorange(req, len, r);
 	if (i) {
 		VSLb(req->vsl, SLT_Debug, "RANGE_FAIL line %d", i);
 		http_Unset(req->resp, H_Content_Length);
diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index f479498..ee46e29 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -113,3 +113,60 @@ client c1 {
 } -run
 
 varnish v1 -expect s_resp_bodybytes == 401
+
+# Test Range streaming with streaming objects with C-L
+
+server s1 -repeat 2 {
+	rxreq
+	txresp -nolen -hdr "Content-Length: 100"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	sema r1 sync 2
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		if (bereq.url == "/2") {
+			set beresp.do_gzip = true;
+		}
+	}
+}
+
+client c1 {
+	# Open ended range to force C-L usage
+	txreq -url /1 \
+		-hdr "Range: bytes=20-" \
+		-hdr "Connection: close"
+	rxresphdrs
+	expect resp.status == 206
+	expect resp.http.Content-Range == "bytes 20-99/100"
+	recv 10
+	sema r1 sync 2
+	recv 70
+	expect_close
+} -run
+
+delay .1
+
+client c1 {
+	# Closed C-L because we cannot use C-L
+	txreq -url /2 \
+		-hdr "Range: bytes=20-29" \
+		-hdr "Connection: close" \
+		-hdr "Accept-encoding: gzip"
+	rxresphdrs
+	expect resp.status == 206
+	expect resp.http.Content-Range == "bytes 20-29/*"
+	expect resp.http.Content-Length == 10
+	sema r1 sync 2
+	recv 10
+	expect_close
+} -run



From phk at FreeBSD.org  Tue May  5 12:42:32 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 14:42:32 +0200
Subject: [master] 16bc126 Try to make the new tests more robust
Message-ID: <E1YpcBE-00051l-EH@project.varnish-software.com>


commit 16bc12623abb66cf969f86b1a3c439e0d122289d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 12:42:17 2015 +0000

    Try to make the new tests more robust

diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index ee46e29..bab80f0 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -124,10 +124,10 @@ server s1 -repeat 2 {
 	send "0123456789"
 	send "0123456789"
 	send "0123456789"
-	sema r1 sync 2
 	send "0123456789"
 	send "0123456789"
 	send "0123456789"
+	sema r1 sync 2
 	send "0123456789"
 	send "0123456789"
 } -start
@@ -159,14 +159,14 @@ delay .1
 client c1 {
 	# Closed C-L because we cannot use C-L
 	txreq -url /2 \
-		-hdr "Range: bytes=20-29" \
+		-hdr "Range: bytes=2-5" \
 		-hdr "Connection: close" \
 		-hdr "Accept-encoding: gzip"
 	rxresphdrs
 	expect resp.status == 206
-	expect resp.http.Content-Range == "bytes 20-29/*"
-	expect resp.http.Content-Length == 10
+	expect resp.http.Content-Range == "bytes 2-5/*"
+	expect resp.http.Content-Length == 4
 	sema r1 sync 2
-	recv 10
+	recv 4
 	expect_close
 } -run



From phk at FreeBSD.org  Tue May  5 12:50:38 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 14:50:38 +0200
Subject: [master] cd0329b Take another shot at stability in this case
Message-ID: <E1YpcJ4-0007ja-DV@project.varnish-software.com>


commit cd0329bc149c9593450de119ebea1b3f1f0418ee
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 12:50:11 2015 +0000

    Take another shot at stability in this case

diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index bab80f0..434b826 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -116,7 +116,7 @@ varnish v1 -expect s_resp_bodybytes == 401
 
 # Test Range streaming with streaming objects with C-L
 
-server s1 -repeat 2 {
+server s1 {
 	rxreq
 	txresp -nolen -hdr "Content-Length: 100"
 	send "0123456789"
@@ -130,6 +130,20 @@ server s1 -repeat 2 {
 	sema r1 sync 2
 	send "0123456789"
 	send "0123456789"
+
+	rxreq
+	txresp -nolen -hdr "Content-Length: 100"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	send "0123456789"
+	sema r2 sync 2
+	send "0123456789"
+	send "0123456789"
 } -start
 
 varnish v1 -vcl+backend {
@@ -166,7 +180,7 @@ client c1 {
 	expect resp.status == 206
 	expect resp.http.Content-Range == "bytes 2-5/*"
 	expect resp.http.Content-Length == 4
-	sema r1 sync 2
+	sema r2 sync 2
 	recv 4
 	expect_close
 } -run



From phk at FreeBSD.org  Tue May  5 13:03:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 15:03:04 +0200
Subject: [master] e2e4542 Give up synchronizing the last test.
Message-ID: <E1YpcV6-0000iC-VK@project.varnish-software.com>


commit e2e45424a36f6de70850cbede2713bb8a3a0783f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 13:02:51 2015 +0000

    Give up synchronizing the last test.

diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index 434b826..e9f6145 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -141,7 +141,6 @@ server s1 {
 	send "0123456789"
 	send "0123456789"
 	send "0123456789"
-	sema r2 sync 2
 	send "0123456789"
 	send "0123456789"
 } -start
@@ -174,13 +173,9 @@ client c1 {
 	# Closed C-L because we cannot use C-L
 	txreq -url /2 \
 		-hdr "Range: bytes=2-5" \
-		-hdr "Connection: close" \
 		-hdr "Accept-encoding: gzip"
-	rxresphdrs
+	rxresp
 	expect resp.status == 206
 	expect resp.http.Content-Range == "bytes 2-5/*"
 	expect resp.http.Content-Length == 4
-	sema r2 sync 2
-	recv 4
-	expect_close
 } -run



From phk at FreeBSD.org  Tue May  5 13:08:37 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 15:08:37 +0200
Subject: [master] 00da3e1 I guess none of these two test can be reliably
 sync'ed across platforms, but I'm not quite sure I know why...
Message-ID: <E1YpcaT-0001Up-1B@project.varnish-software.com>


commit 00da3e166b80a5f68e3710ee05bd536959c87089
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 13:08:08 2015 +0000

    I guess none of these two test can be reliably sync'ed across
    platforms, but I'm not quite sure I know why...

diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index e9f6145..ee75995 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -127,7 +127,6 @@ server s1 {
 	send "0123456789"
 	send "0123456789"
 	send "0123456789"
-	sema r1 sync 2
 	send "0123456789"
 	send "0123456789"
 
@@ -161,9 +160,7 @@ client c1 {
 	rxresphdrs
 	expect resp.status == 206
 	expect resp.http.Content-Range == "bytes 20-99/100"
-	recv 10
-	sema r1 sync 2
-	recv 70
+	recv 80
 	expect_close
 } -run
 



From phk at FreeBSD.org  Tue May  5 13:30:25 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 05 May 2015 15:30:25 +0200
Subject: [master] 5aaf7db Like any test involving streaming this looks like it
 will become a unstable beast :-/
Message-ID: <E1YpcvZ-0003cB-L1@project.varnish-software.com>


commit 5aaf7dba329132b3a979c541065c9fd3796d6b8a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 5 13:29:55 2015 +0000

    Like any test involving streaming this looks like it will become
    a unstable beast :-/

diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index ee75995..0170b22 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -128,6 +128,7 @@ server s1 {
 	send "0123456789"
 	send "0123456789"
 	send "0123456789"
+	delay 2
 	send "0123456789"
 
 	rxreq
@@ -141,6 +142,7 @@ server s1 {
 	send "0123456789"
 	send "0123456789"
 	send "0123456789"
+	delay 2
 	send "0123456789"
 } -start
 



From lkarsten at varnish-software.com  Tue May  5 14:10:41 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 05 May 2015 16:10:41 +0200
Subject: [master] 42c33b0 Prepare skeleton changes and upgrade sections.
Message-ID: <E1YpdYX-0008Aj-2A@project.varnish-software.com>


commit 42c33b0906ef53c3bf28722303ad06235e869d27
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 5 12:55:04 2015 +0200

    Prepare skeleton changes and upgrade sections.
    
    Time to start writing down what will be part of 4.1.

diff --git a/doc/changes.rst b/doc/changes.rst
index 8b26297..3caf9f0 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -1,3 +1,13 @@
+====================================
+Varnish Cache 4.1.0-dev (unreleased)
+====================================
+
+This is a placeholder entry for the upcoming 4.1.0 prereleases.
+
+Changes between 4.0 and 4.1 are numerous. Please read the upgrade
+section in the documentation for a general overview.
+
+
 ============================================
 Changes from 4.0.3-rc3 to 4.0.3 (2015-02-17)
 ============================================
diff --git a/doc/sphinx/whats-new/changes.rst b/doc/sphinx/whats-new/changes.rst
index 274c9f5..66c9255 100644
--- a/doc/sphinx/whats-new/changes.rst
+++ b/doc/sphinx/whats-new/changes.rst
@@ -1,17 +1,99 @@
 .. _whatsnew_changes:
 
-Changes in Varnish 4
-====================
+Changes in Varnish 4.1 (unreleased)
+===================================
 
-Varnish 4 is quite an extensive update to Varnish 3, with some very big improvements to central parts of varnish.
+Varnish 4.1 is the continuation of the new streaming architecture seen in Varnish 4.0.
 
-Client/backend split
---------------------
-In the past, Varnish has fetched the content from the backend in the same
-thread as the client request.In Varnish 4 we have  split the client and backend code into separate trheads allowing for some much requested improvements.
-This split allows Varnish to refresh content in the background while serving
-stale content quickly to the client.
 
-This split has also necessitated a change of the VCL-functions, in particular functionality has moved from the old `vcl_fetch` method to the two new methods `vcl_backend_fetch` and `vcl_backend_response`.
+Proactive security features
+===========================
+
+New in 4.1 is support for different kinds of privilege seperation methods,
+collectively described as jails.
+
+On most systems, the Varnish parent process will now drop effective privileges
+to normal user mode when not doing operations needing special access.
+
+The Varnish worker child should now be run as a separate `vcache` user.
+
+``varnishlog``, ``varnishncsa`` and other Varnish shared log utilities now must
+be run in a context with `varnish` group membership.
+
+
+Warm and cold VCL configurations
+================================
+
+Traditionally Varnish have had the concept of active and inactive loaded VCLs.
+Any loaded VCL lead to state being kept, and a separate set of health checks (if
+configured) were being run against the backends.
+
+To avoid the extra state and backend polling, a loaded VCL is now either warm
+or cold. Runtime state (incl. backend counters) and health checks are not
+present for cold VCLs.
+
+A warm VCL will automatically be set to cold after `vcl_cooldown` seconds.
+
+Output from `vcl.list`::
+
+    varnish> vcl.list
+    200
+    available  auto/warm       0 boot
+    available  auto/warm       0 62f5275f-a937-4df9-9fbb-c12336bdfdb8
+
+
+A single VCL's state can be chanced with the `vcl.state` call in ``varnishadm``::
+
+    vcl.state <configname> [auto|cold|warm]
+        Force the state of the named configuration.
+
+Example::
+
+
+    varnish> vcl.state 62f5275f-a937-4df9-9fbb-c12336bdfdb8 cold
+    200
+
+    varnish> vcl.list
+    200
+    available  auto/warm       0 boot
+    available  auto/cold       0 62f5275f-a937-4df9-9fbb-c12336bdfdb8
+
+
+VMOD writers should read up on the new vcl_event system to release unnecessary
+state when a VCL is transitioned to cold.
+
+
+PROXY protocol support
+======================
+
+Socket support for PROXY protocol connections has been added. PROXY defines a
+short preamble on the TCP connection where (usually) a SSL/TLS terminating
+proxy can signal the real client address.
+
+The ``-a`` startup argument syntax has been expanded to allow for this::
+
+    $ varnishd -f /etc/varnish/default.vcl -a :6081,proxy at 127.0.0.1:6086
+
+
+Both PROXY1 and PROXY2 protocols are supported on the resulting listening
+socket.
+
+
+VMOD backends
+=============
+
+Not yet documented.
+
+
+Surrogate keys
+==============
+
+Not yet documented.
+
+Other noteworthy small changes
+==============================
+
+* Varnish will now use the ``stale-while-revalidate`` defined in RFC5861 to set object grace time.
+* Varnish will now discard remaining/older open backend connections when a failing connection is found.
+* -smalloc storage is now recommended over -sfile on Linux systems.
 
-.. XXX:Here would an updated flow-diagram over functions be great. benc
diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst
index c9d9130..2b82bb6 100644
--- a/doc/sphinx/whats-new/index.rst
+++ b/doc/sphinx/whats-new/index.rst
@@ -1,19 +1,21 @@
 .. _whats-new-index:
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%
-What's new in Varnish 4.0
+What's new in Varnish 4.1
 %%%%%%%%%%%%%%%%%%%%%%%%%%
 
-This section describes the changes that have been made for Varnish 4. The
-first subsection describes overarching changes that have gone into
+This section describes the changes that have been made for Varnish 4.1.
+
+The first subsection describes overarching changes that have gone into
 Varnish 4.0, while the second subsection describes changes you need to make to
-your current configuration (assuming you are on Varnish 3.x) as well as any changes in behaviour that you need to be aware of and take
-into consideration when upgrading.
+your current configuration if you come from Varnish 3.x.
 
-.. XXX:Heavy change of meaning above! benc
+The text also considers changes in behaviour that you need to be aware of and
+take into consideration when upgrading.
 
 .. toctree::
    :maxdepth: 2
 
    changes
    upgrading
+   upgrade-4.0
diff --git a/doc/sphinx/whats-new/upgrade-4.0.rst b/doc/sphinx/whats-new/upgrade-4.0.rst
new file mode 100644
index 0000000..d1df915
--- /dev/null
+++ b/doc/sphinx/whats-new/upgrade-4.0.rst
@@ -0,0 +1,248 @@
+.. _whatsnew_upgrading:
+
+%%%%%%%%%%%%%%%%%%%%%%%%
+Upgrading to Varnish 4.0
+%%%%%%%%%%%%%%%%%%%%%%%%
+
+Changes to VCL
+==============
+
+The backend fetch parts of VCL have changed in Varnish 4. We've tried to
+compile a list of changes needed to upgrade here.
+
+Version statement
+~~~~~~~~~~~~~~~~~
+
+To make sure that people have upgraded their VCL to the current
+version, Varnish now requires the first line of VCL to indicate the
+VCL version number::
+
+    vcl 4.0;
+
+req.request is now req.method
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To align better with RFC naming, `req.request` has been renamed to
+`req.method`.
+
+vcl_fetch is now vcl_backend_response
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Directors have been moved to the vmod_directors
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To make directors (backend selection logic) easier to extend, the
+directors are now defined in loadable VMODs.
+
+Setting a backend for future fetches in `vcl_recv` is now done as follows::
+
+    sub vcl_init {
+        new cluster1 = directors.round_robin();
+        cluster1.add_backend(b1, 1.0);
+        cluster1.add_backend(b2, 1.0);
+    }
+
+    sub vcl_recv {
+        set req.backend_hint = cluster1.backend();
+    }
+
+Note the extra `.backend()` needed after the director name.
+
+Use the hash director as a client director
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Since the client director was already a special case of the hash director, it
+has been removed, and you should use the hash director directly::
+
+    sub vcl_init {
+        new h = directors.hash();
+        h.add_backend(b1, 1);
+        h.add_backend(b2, 1);
+    }
+
+    sub vcl_recv {
+        set req.backend_hint = h.backend(client.identity);
+    }
+
+vcl_error is now vcl_backend_error
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To make a distinction between internally generated errors and
+VCL synthetic responses, `vcl_backend_error` will be called when
+varnish encounters an error when trying to fetch an object.
+
+error() is now synth()
+~~~~~~~~~~~~~~~~~~~~~~
+
+And you must explicitly return it::
+
+    return (synth(999, "Response"));
+
+Synthetic responses in vcl_synth
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Setting headers on synthetic response bodies made in vcl_synth are now done on
+resp.http instead of obj.http.
+
+The synthetic keyword is now a function::
+
+    if (resp.status == 799) {
+        set resp.status = 200;
+        set resp.http.Content-Type = "text/plain; charset=utf-8";
+        synthetic("You are " + client.ip);
+        return (deliver);
+    }
+
+obj in vcl_error replaced by beresp in vcl_backend_error
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To better represent a the context in which it is called, you
+should now use `beresp.*` vcl_backend_error, where you used to
+use `obj.*` in `vcl_error`.
+
+hit_for_pass objects are created using beresp.uncacheable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Example::
+
+    sub vcl_backend_response {
+        if (beresp.http.X-No-Cache) {
+            set beresp.uncacheable = true;
+            set beresp.ttl = 120s;
+            return (deliver);
+        }
+    }
+
+req.* not available in vcl_backend_response
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+req.* used to be available in `vcl_fetch`, but after the split of
+functionality, you only have 'bereq.*' in `vcl_backend_response`.
+
+vcl_* reserved
+~~~~~~~~~~~~~~
+
+Any custom-made subs cannot be named 'vcl_*' anymore. This namespace
+is reserved for builtin subs.
+
+req.backend.healthy replaced by std.healthy(req.backend_hint)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Remember to import the std module if you're not doing so already.
+
+client.port, and server.port replaced by respectively std.port(client.ip) and std.port(server.ip)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`client.ip` and `server.ip` are now proper datatypes, which renders
+as an IP address by default. You need to use the `std.port()`
+function to get the port number.
+
+Invalidation with purge
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Cache invalidation with purges is now done via `return(purge)` from `vcl_recv`.
+The `purge;` keyword has been retired.
+
+obj is now read-only
+~~~~~~~~~~~~~~~~~~~~
+
+`obj` is now read-only.  `obj.last_use` has been retired.
+
+Some return values have been replaced
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Apart from the new `synth` return value described above, the
+following has changed:
+
+ - `vcl_recv` must now return `hash` instead of `lookup`
+ - `vcl_hash` must now return `lookup` instead of `hash`
+ - `vcl_pass` must now return `fetch` instead of `pass`
+
+
+Backend restarts are now retry
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In 3.0 it was possible to do `return(restart)` after noticing that
+the backend response was wrong, to change to a different backend.
+
+This is now called `return(retry)`, and jumps back up to `vcl_backend_fetch`.
+
+This only influences the backend fetch thread, client-side handling is not affected.
+
+
+default/builtin VCL changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The VCL code that is appended to user-configured VCL automatically
+is now called the builtin VCL. (previously default.vcl)
+
+The builtin VCL now honors Cache-Control: no-cache (and friends)
+to indicate uncacheable content from the backend.
+
+
+The `remove` keyword is gone
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Replaced by `unset`.
+
+
+X-Forwarded-For is now set before vcl_recv
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In many cases, people unintentionally removed X-Forwarded-For when
+implementing their own vcl_recv. Therefore it has been moved to before
+vcl_recv, so if you don't want an IP added to it, you should remove it
+in vcl_recv.
+
+
+Changes to existing parameters
+==============================
+
+session_linger
+~~~~~~~~~~~~~~
+`session_linger` has been renamed to `timeout_linger` and it is in
+seconds now (previously was milliseconds).
+
+sess_timeout
+~~~~~~~~~~~~
+`sess_timeout` has been renamed to `timeout_idle`.
+
+sess_workspace
+~~~~~~~~~~~~~~
+
+In 3.0 it was often necessary to increase `sess_workspace` if a
+lot of VMODs, complex header operations or ESI were in use.
+
+This is no longer necessary, because ESI scratch space happens
+elsewhere in 4.0.
+
+If you are using a lot of VMODs,  you may need to increase
+either `workspace_backend` and `workspace_client` based on where
+your VMOD is doing its work.
+
+thread_pool_purge_delay
+~~~~~~~~~~~~~~~~~~~~~~~
+`thread_pool_purge_delay` has been renamed to `thread_pool_destroy_delay`
+and it is in seconds now (previously was milliseconds).
+
+thread_pool_add_delay and thread_pool_fail_delay
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+They are in seconds now (previously were milliseconds).
+
+New parameters since 3.0
+========================
+
+vcc_allow_inline_c
+~~~~~~~~~~~~~~~~~~
+
+You can now completely disable inline C in your VCL, and it is
+disabled by default.
+
+Other changes
+=============
+
+New log filtering
+~~~~~~~~~~~~~~~~~
+
+The logging framework has a new filtering language, which means that
+the -m switch has been replaced with a new -q switch.  See
+:ref:`vsl-query(7)` for more information about the new query language.
diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index c882b29..058aa5e 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -1,248 +1,74 @@
-.. _whatsnew_upgrading:
+.. _whatsnew_upgrade41:
 
-%%%%%%%%%%%%%%%%%%%%%%
-Upgrading to Varnish 4
-%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+Upgrading to Varnish 4.1 (unreleased)
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 Changes to VCL
 ==============
 
-The backend fetch parts of VCL have changed in Varnish 4. We've tried to
-compile a list of changes needed to upgrade here.
+Not documented yet.
 
-Version statement
-~~~~~~~~~~~~~~~~~
+Version statement is kept
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To make sure that people have upgraded their VCL to the current
-version, Varnish now requires the first line of VCL to indicate the
-VCL version number::
+The VCL syntax has not chanced significantly, and as such the Varnish 4.0
+version marker is kept for Varnish 4.1.
 
-    vcl 4.0;
-
-req.request is now req.method
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To align better with RFC naming, `req.request` has been renamed to
-`req.method`.
-
-vcl_fetch is now vcl_backend_response
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Directors have been moved to the vmod_directors
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To make directors (backend selection logic) easier to extend, the
-directors are now defined in loadable VMODs.
-
-Setting a backend for future fetches in `vcl_recv` is now done as follows::
-
-    sub vcl_init {
-        new cluster1 = directors.round_robin();
-        cluster1.add_backend(b1, 1.0);
-        cluster1.add_backend(b2, 1.0);
-    }
-
-    sub vcl_recv {
-        set req.backend_hint = cluster1.backend();
-    }
-
-Note the extra `.backend()` needed after the director name.
-
-Use the hash director as a client director
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Since the client director was already a special case of the hash director, it
-has been removed, and you should use the hash director directly::
-
-    sub vcl_init {
-        new h = directors.hash();
-        h.add_backend(b1, 1);
-        h.add_backend(b2, 1);
-    }
-
-    sub vcl_recv {
-        set req.backend_hint = h.backend(client.identity);
-    }
-
-vcl_error is now vcl_backend_error
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To make a distinction between internally generated errors and
-VCL synthetic responses, `vcl_backend_error` will be called when
-varnish encounters an error when trying to fetch an object.
-
-error() is now synth()
-~~~~~~~~~~~~~~~~~~~~~~
-
-And you must explicitly return it::
-
-    return (synth(999, "Response"));
-
-Synthetic responses in vcl_synth
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Setting headers on synthetic response bodies made in vcl_synth are now done on
-resp.http instead of obj.http.
-
-The synthetic keyword is now a function::
-
-    if (resp.status == 799) {
-        set resp.status = 200;
-        set resp.http.Content-Type = "text/plain; charset=utf-8";
-        synthetic("You are " + client.ip);
-        return (deliver);
-    }
-
-obj in vcl_error replaced by beresp in vcl_backend_error
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To better represent a the context in which it is called, you
-should now use `beresp.*` vcl_backend_error, where you used to
-use `obj.*` in `vcl_error`.
-
-hit_for_pass objects are created using beresp.uncacheable
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Example::
-
-    sub vcl_backend_response {
-        if (beresp.http.X-No-Cache) {
-            set beresp.uncacheable = true;
-            set beresp.ttl = 120s;
-            return (deliver);
-        }
-    }
-
-req.* not available in vcl_backend_response
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-req.* used to be available in `vcl_fetch`, but after the split of
-functionality, you only have 'bereq.*' in `vcl_backend_response`.
-
-vcl_* reserved
-~~~~~~~~~~~~~~
-
-Any custom-made subs cannot be named 'vcl_*' anymore. This namespace
-is reserved for builtin subs.
-
-req.backend.healthy replaced by std.healthy(req.backend_hint)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Remember to import the std module if you're not doing so already.
-
-client.port, and server.port replaced by respectively std.port(client.ip) and std.port(server.ip)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-`client.ip` and `server.ip` are now proper datatypes, which renders
-as an IP address by default. You need to use the `std.port()`
-function to get the port number.
-
-Invalidation with purge
-~~~~~~~~~~~~~~~~~~~~~~~
-
-Cache invalidation with purges is now done via `return(purge)` from `vcl_recv`.
-The `purge;` keyword has been retired.
-
-obj is now read-only
-~~~~~~~~~~~~~~~~~~~~
-
-`obj` is now read-only.  `obj.last_use` has been retired.
-
-Some return values have been replaced
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Apart from the new `synth` return value described above, the
-following has changed:
-
- - `vcl_recv` must now return `hash` instead of `lookup`
- - `vcl_hash` must now return `lookup` instead of `hash`
- - `vcl_pass` must now return `fetch` instead of `pass`
-
-
-Backend restarts are now retry
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-In 3.0 it was possible to do `return(restart)` after noticing that
-the backend response was wrong, to change to a different backend.
-
-This is now called `return(retry)`, and jumps back up to `vcl_backend_fetch`.
-
-This only influences the backend fetch thread, client-side handling is not affected.
-
-
-default/builtin VCL changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The VCL code that is appended to user-configured VCL automatically
-is now called the builtin VCL. (previously default.vcl)
-
-The builtin VCL now honors Cache-Control: no-cache (and friends)
-to indicate uncacheable content from the backend.
+One of the initial lines in a Varnish 4.1 VCL should read::
 
+    vcl 4.0;
 
-The `remove` keyword is gone
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Remote address accessors
+~~~~~~~~~~~~~~~~~~~~~~~~
 
-Replaced by `unset`.
+New in 4.1 is the `local.ip` and `remote.ip` representing the (local) TCP
+connection endpoints.
 
+With PROXY listeners the `server.ip` and `client.ip` are set from the PROXY
+preamble. On normal HTTP listeners the behaviour is unchanged.
 
-X-Forwarded-For is now set before vcl_recv
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-In many cases, people unintentionally removed X-Forwarded-For when
-implementing their own vcl_recv. Therefore it has been moved to before
-vcl_recv, so if you don't want an IP added to it, you should remove it
-in vcl_recv.
+Management interface
+====================
 
+The management interface enabled with ``-M`` used to support the telnet
+protocol.
 
-Changes to existing parameters
-==============================
+Support for telnet control sequences have been retired. Replacements are netcat
+or (preferred) ``varnishadm``.
 
-session_linger
-~~~~~~~~~~~~~~
-`session_linger` has been renamed to `timeout_linger` and it is in
-seconds now (previously was milliseconds).
 
-sess_timeout
-~~~~~~~~~~~~
-`sess_timeout` has been renamed to `timeout_idle`.
+Runtime users and groups
+========================
 
-sess_workspace
-~~~~~~~~~~~~~~
+With the new jail support, an additional runtime user (`vcache`) should be used
+for the Varnish worker child process.
 
-In 3.0 it was often necessary to increase `sess_workspace` if a
-lot of VMODs, complex header operations or ESI were in use.
+Additionally, the ``varnishlog``, ``varnishncsa`` and other Varnish shared log
+utilities must now be run in a context with `varnish` group membership.
 
-This is no longer necessary, because ESI scratch space happens
-elsewhere in 4.0.
 
-If you are using a lot of VMODs,  you may need to increase
-either `workspace_backend` and `workspace_client` based on where
-your VMOD is doing its work.
+Protocol support
+================
 
-thread_pool_purge_delay
-~~~~~~~~~~~~~~~~~~~~~~~
-`thread_pool_purge_delay` has been renamed to `thread_pool_destroy_delay`
-and it is in seconds now (previously was milliseconds).
+Support for HTTP/0.9 on the client side has been retired.
 
-thread_pool_add_delay and thread_pool_fail_delay
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-They are in seconds now (previously were milliseconds).
 
-New parameters since 3.0
-========================
+Changes to parameters
+=====================
 
-vcc_allow_inline_c
-~~~~~~~~~~~~~~~~~~
+`vcl_cooldown` is new, and decides how long time a VCL is kept warm after being
+replaced as the active VCL.
 
-You can now completely disable inline C in your VCL, and it is
-disabled by default.
+The following parameters have been retired:
 
-Other changes
-=============
+* `group` (security rewamp)
+* `group_cc` (security rewamp)
+* `listen_address` (security rewamp)
+* `pool_vbc`
+* `timeout_req` - merged with `timeout_idle`.
+* `user` (security rewamp)
 
-New log filtering
-~~~~~~~~~~~~~~~~~
+Minor changes of default values on `workspace_session` and `vsl_mask`.
 
-The logging framework has a new filtering language, which means that
-the -m switch has been replaced with a new -q switch.  See
-:ref:`vsl-query(7)` for more information about the new query language.



From lkarsten at varnish-software.com  Tue May  5 14:10:41 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 05 May 2015 16:10:41 +0200
Subject: [master] f12e24e Remove extra whitespace in generated C code.
Message-ID: <E1YpdYX-0008Aq-3Q@project.varnish-software.com>


commit f12e24ef3bf63193e5cf1d250046698db3329775
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 5 16:08:55 2015 +0200

    Remove extra whitespace in generated C code.

diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index f651ed7..4717148 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -348,7 +348,7 @@ EmitInitFini(const struct vcc *tl)
 	Fc(tl, 0, "\t\treturn(VGC_Load(ctx));\n");
 	Fc(tl, 0, "\tif (ev == VCL_EVENT_DISCARD)\n");
 	Fc(tl, 0, "\t\treturn(VGC_Discard(ctx));\n");
-	Fc(tl, 0, "\t\n");
+	Fc(tl, 0, "\n");
 	VTAILQ_FOREACH(p, &tl->inifin, list) {
 		AZ(VSB_finish(p->event));
 		if (VSB_len(p->event))
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index 74713b8..0fd45ec 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -78,7 +78,7 @@ vcc_Var_Wildcard(struct vcc *tl, const struct token *t, const struct symbol *wc)
 
 	bprintf(buf, "&VGC_%s_%s", vh->rname, cnam);
 	v->rname = TlDup(tl, buf);
-	bprintf(buf, "VRT_SetHdr(ctx, %s, ", v->rname);
+	bprintf(buf, "VRT_SetHdr(ctx, %s,", v->rname);
 	v->lname = TlDup(tl, buf);
 
 	sym = VCC_AddSymbolTok(tl, t, SYM_VAR);



From lkarsten at varnish-software.com  Tue May  5 14:25:48 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 05 May 2015 16:25:48 +0200
Subject: [master] d135985 Use PROXY syntax.
Message-ID: <E1YpdnA-0003dv-JW@project.varnish-software.com>


commit d13598575f6397376b1796d9952af8aff792d52d
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 5 16:24:06 2015 +0200

    Use PROXY syntax.
    
    This is obviously different than in the original patch, a change
    which I seem to have missed.
    
    Spotted by Dridi.

diff --git a/doc/sphinx/whats-new/changes.rst b/doc/sphinx/whats-new/changes.rst
index 66c9255..c5d181b 100644
--- a/doc/sphinx/whats-new/changes.rst
+++ b/doc/sphinx/whats-new/changes.rst
@@ -72,7 +72,7 @@ proxy can signal the real client address.
 
 The ``-a`` startup argument syntax has been expanded to allow for this::
 
-    $ varnishd -f /etc/varnish/default.vcl -a :6081,proxy at 127.0.0.1:6086
+    $ varnishd -f /etc/varnish/default.vcl -a :6081 -a 127.0.0.1:6086,PROXY
 
 
 Both PROXY1 and PROXY2 protocols are supported on the resulting listening



From lkarsten at varnish-software.com  Tue May  5 14:39:32 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 05 May 2015 16:39:32 +0200
Subject: [master] cf08566 Document new -a format.
Message-ID: <E1Ype0S-0007Oi-Ar@project.varnish-software.com>


commit cf08566247669e6eaec251ba7932a3bd13d612a1
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 5 16:37:30 2015 +0200

    Document new -a format.
    
    With PROXY protocol support, the behaviour of varnishd startup argument
    -a changed.
    
    Reflect this in the man page and adjacent documentation.

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index a9cf7d5..f6be70f 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -15,7 +15,7 @@ HTTP accelerator daemon
 SYNOPSIS
 ========
 
-varnishd [-a address[:port]] [-b host[:port]] [-C] [-d] [-f config] [-F] [-g group] [-h type[,options]] [-i identity] [-l shl[,free[,fill]]] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...] [-s [name=]kind[,options]] [-S secret-file] [-T address[:port]] [-t ttl] [-u user] [-V]
+varnishd [-a address[:port][,PROTO]] [-b host[:port]] [-C] [-d] [-f config] [-F] [-g group] [-h type[,options]] [-i identity] [-l shl[,free[,fill]]] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...] [-s [name=]kind[,options]] [-S secret-file] [-T address[:port]] [-t ttl] [-u user] [-V]
 
 DESCRIPTION
 ===========
@@ -29,16 +29,17 @@ satisfy future requests for the same document.
 OPTIONS
 =======
 
--a <address[:port][,address[:port][...]>
+-a <address[:port][,PROTO]>
 
   Listen for client requests on the specified address and port.  The
-  address can be a host name (?localhost?), an IPv4 dotted-quad
-  (?127.0.0.1?), or an IPv6 address enclosed in square brackets
-  (?[::1]?).  If address is not specified, varnishd will listen on all
+  address can be a host name ("localhost"), an IPv4 dotted-quad
+  ("127.0.0.1"), or an IPv6 address enclosed in square brackets
+  ("[::1]").  If address is not specified, varnishd will listen on all
   available IPv4 and IPv6 interfaces.  If port is not specified, the
-  default HTTP port as listed in /etc/services is used.  Multiple
-  listening addresses and ports can be specified as a whitespace or
-  comma -separated list.
+  default HTTP port as listed in /etc/services is used.
+  An additional protocol type can be set for the listening socket with PROTO.
+  Valid protocol types are: HTTP/1 (default), and PROXY.
+  Multiple listening adresses can be specificed by using multiple -a arguments.
 
 -b <host[:port]>
 



From lkarsten at varnish-software.com  Tue May  5 15:03:42 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 05 May 2015 17:03:42 +0200
Subject: [master] 668be11 Update to regular style and prose.
Message-ID: <E1YpeNq-0003Mx-1J@project.varnish-software.com>


commit 668be1114d2407f937e9ea0a4d72f6d1d6c28e37
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 5 17:02:20 2015 +0200

    Update to regular style and prose.
    
    While passing by, update the text and formatting to
    our regular style.

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index f6be70f..91d977a 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -15,12 +15,12 @@ HTTP accelerator daemon
 SYNOPSIS
 ========
 
-varnishd [-a address[:port][,PROTO]] [-b host[:port]] [-C] [-d] [-f config] [-F] [-g group] [-h type[,options]] [-i identity] [-l shl[,free[,fill]]] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...] [-s [name=]kind[,options]] [-S secret-file] [-T address[:port]] [-t ttl] [-u user] [-V]
+varnishd [-a address[:port][,PROTO]] [-b host[:port]] [-C] [-d] [-f config] [-F] [-g group] [-h type[,options]] [-i identity] [-l shl[,free[,fill]]] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...] [-s [name=]kind[,options]] [-S secret-file] [-T address[:port]] [-t TTL] [-u user] [-V]
 
 DESCRIPTION
 ===========
 
-The varnishd daemon accepts HTTP requests from clients, passes them on
+The `varnishd` daemon accepts HTTP requests from clients, passes them on
 to a backend server and caches the returned documents to better
 satisfy future requests for the same document.
 
@@ -31,19 +31,19 @@ OPTIONS
 
 -a <address[:port][,PROTO]>
 
-  Listen for client requests on the specified address and port.  The
+  Listen for client requests on the specified address and port. The
   address can be a host name ("localhost"), an IPv4 dotted-quad
   ("127.0.0.1"), or an IPv6 address enclosed in square brackets
-  ("[::1]").  If address is not specified, varnishd will listen on all
-  available IPv4 and IPv6 interfaces.  If port is not specified, the
-  default HTTP port as listed in /etc/services is used.
+  ("[::1]"). If address is not specified, `varnishd` will listen on all
+  available IPv4 and IPv6 interfaces. If port is not specified, the
+  default HTTP port as listed in ``/etc/services`` is used.
   An additional protocol type can be set for the listening socket with PROTO.
   Valid protocol types are: HTTP/1 (default), and PROXY.
   Multiple listening adresses can be specificed by using multiple -a arguments.
 
 -b <host[:port]>
 
-  Use the specified host as backend server.  If port is not specified,
+  Use the specified host as backend server. If port is not specified,
   the default is 8080.
 
 -C
@@ -55,34 +55,34 @@ OPTIONS
 
   Enables debugging mode: The parent process runs in the foreground
   with a CLI connection on stdin/stdout, and the child process must be
-  started explicitly with a CLI command.  Terminating the parent
+  started explicitly with a CLI command. Terminating the parent
   process will also terminate the child.
 
 -f config
 
   Use the specified VCL configuration file instead of the builtin
   default.  See :ref:`vcl(7)` for details on VCL syntax. When no
-  configuration is supplied varnishd will not start the cache process.
+  configuration is supplied `varnishd` will not start the child process.
 
 -F
 
-  Run in the foreground.
+  Do not fork, run in the foreground.
 
 -g group
 
   Specifies the name of an unprivileged group to which the child
-  process should switch before it starts accepting connections.  This
+  process should switch before it starts accepting connections. This
   is a shortcut for specifying the group run-time parameter.
 
 -h <type[,options]>
 
-  Specifies the hash algorithm.  See `Hash Algorithm Options`_ for a
+  Specifies the hash algorithm. See `Hash Algorithm Options`_ for a
   list of supported algorithms.
 
 -i identity
 
-  Specify the identity of the Varnish server.  This can be accessed
-  using server.identity from VCL
+  Specify the identity of the Varnish server. This can be accessed
+  using ``server.identity`` from VCL.
 
 -j <jail[,jailoptions]>
 
@@ -103,15 +103,15 @@ OPTIONS
 
 -n name
 
-  Specify the name for this instance.  Amonst other things, this name
-  is used to construct the name of the directory in which varnishd
-  keeps temporary files and persistent state.  If the specified name
+  Specify the name for this instance.  Amongst other things, this name
+  is used to construct the name of the directory in which `varnishd`
+  keeps temporary files and persistent state. If the specified name
   begins with a forward slash, it is interpreted as the absolute path
   to the directory which should be used for this purpose.
 
 -P file
 
-  Write the process's PID to the specified file.
+  Write the PID of the process to the specified file.
 
 -p <param=value>
 
@@ -132,22 +132,23 @@ OPTIONS
   Use the specified storage backend, see `Storage Backend Options`_.
 
   This option can be used multiple times to specify multiple storage
-  files. Names are referenced in logs, vcl, statistics, etc.
+  files. Names are referenced in logs, VCL, statistics, etc.
 
 -S file
 
   Path to a file containing a secret used for authorizing access to
-  the management port.
+  the management port. If not provided a new secret will be drawn
+  from the system PRNG.
 
 -T <address[:port]>
 
-  Offer a management interface on the specified address and port.  See
+  Offer a management interface on the specified address and port. See
   `Management Interface`_ for a list of management commands.
 
--t ttl
+-t TTL
 
-  Specifies a hard minimum time to live for cached documents. This is
-  a shortcut for specifying the default_ttl run-time parameter.
+  Specifies the default time to live (TTL) for cached objects. This is
+  a shortcut for specifying the *default_ttl* run-time parameter.
 
 -u user
 
@@ -235,7 +236,7 @@ specific options. Available jails are:
 
 -j solaris
 
-  Reduce privileges(5) for varnishd and sub-process to the minimally
+  Reduce privileges(5) for `varnishd` and sub-process to the minimally
   required set. Only available on platforms which have the setppriv(2)
   call.
 
@@ -265,7 +266,7 @@ specific options. Available jails are:
 Management Interface
 --------------------
 
-If the -T option was specified, varnishd will offer a command-line
+If the -T option was specified, `varnishd` will offer a command-line
 management interface on the specified address and port.  The
 recommended way of connecting to the command-line management interface
 is through varnishadm(1).
@@ -281,13 +282,13 @@ Run Time Parameter Flags
 ------------------------
 
 Runtime parameters are marked with shorthand flags to avoid repeating
-the same text over and over in the table below.  The meaning of the
+the same text over and over in the table below. The meaning of the
 flags are:
 
 * `experimental`
 
   We have no solid information about good/bad/optimal values for this
-  parameter.  Feedback with experience and observations are most
+  parameter. Feedback with experience and observations are most
   welcome.
 
 * `delayed`
@@ -314,7 +315,7 @@ flags are:
 
 * `only_root`
 
-  Only works if varnishd is running as root.
+  Only works if `varnishd` is running as root.
 
 Default Value Exceptions on 32 bit Systems
 ------------------------------------------
@@ -369,7 +370,7 @@ SEE ALSO
 HISTORY
 =======
 
-The varnishd daemon was developed by Poul-Henning Kamp in cooperation
+The `varnishd` daemon was developed by Poul-Henning Kamp in cooperation
 with Verdens Gang AS and Varnish Software.
 
 This manual page was written by Dag-Erling Sm?rgrav with updates by
@@ -382,4 +383,4 @@ COPYRIGHT
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
-* Copyright (c) 2007-2014 Varnish Software AS
+* Copyright (c) 2007-2015 Varnish Software AS



From phk at FreeBSD.org  Thu May  7 07:08:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 07 May 2015 09:08:12 +0200
Subject: [master] 86300e2 Remove some debugging, polish
Message-ID: <E1YqFum-00014h-TH@project.varnish-software.com>


commit 86300e248911e50b174ad8dcf4a60890669d4e16
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 6 20:59:33 2015 +0000

    Remove some debugging, polish

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 54422bc..2041a74 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -985,7 +985,7 @@ size_t V1L_Write(const struct worker *w, const void *ptr, ssize_t len);
 task_func_t VPX_Proto_Sess;
 
 /* cache_range.c [VRG] */
-void VRG_dorange(struct req *req, struct busyobj *bo, const char *r);
+void VRG_dorange(struct req *req, const struct busyobj *bo, const char *r);
 
 /* cache_req.c */
 struct req *Req_New(const struct worker *, struct sess *);
diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index 3bb4753..fcff78a 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -172,7 +172,7 @@ vrg_dorange(struct req *req, ssize_t len, const char *r)
 }
 
 void
-VRG_dorange(struct req *req, struct busyobj *bo, const char *r)
+VRG_dorange(struct req *req, const struct busyobj *bo, const char *r)
 {
 	ssize_t len;
 	int i;
@@ -182,14 +182,9 @@ VRG_dorange(struct req *req, struct busyobj *bo, const char *r)
 	assert(http_IsStatus(req->resp, 200));
 
 	/* We must snapshot the length if we're streaming from the backend */
-	if (bo != NULL) {
+	if (bo != NULL)
 		len = http_GetContentLength(bo->beresp);
-VSLb(req->vsl, SLT_Debug, "UUU 1 %jd", (intmax_t)len);
-#if 0
-		if (len < 0)
-			len = VBO_waitlen(req->wrk, bo, -1);
-#endif
-	} else
+	else
 		len = ObjGetLen(req->wrk, req->objcore);
 
 	i = vrg_dorange(req, len, r);



From phk at FreeBSD.org  Thu May  7 07:08:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 07 May 2015 09:08:12 +0200
Subject: [master] 26c460c Turn ESI_Deliver() inside-out,
 so that we iterate over the storage, consuming ESI string as we go,
 rather than the other way around.
Message-ID: <E1YqFum-00014n-UU@project.varnish-software.com>


commit 26c460c4afb166dc2361cb71b487b2c1961cb9a3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 6 21:25:19 2015 +0000

    Turn ESI_Deliver() inside-out, so that we iterate over the storage,
    consuming ESI string as we go, rather than the other way around.

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index d8edb91..92b7553 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -275,161 +275,197 @@ static const uint8_t gzip_hdr[] = {
 	0x02, 0x03
 };
 
-void
-ESI_Deliver(struct req *req)
+struct ecx {
+	uint8_t		*p;
+	uint8_t		*e;
+
+	int		state;
+	ssize_t		l;
+
+	int		isgzip;
+};
+
+static void
+esi_deliver(struct req *req, struct ecx *ecx, const uint8_t *pp, ssize_t sl)
 {
-	uint8_t *p, *e, *q, *r;
-	ssize_t l, l2, l_icrc = 0;
+	uint8_t *q, *r;
+	ssize_t l_icrc = 0;
 	uint32_t icrc = 0;
 	uint8_t tailbuf[8 + 5];
-	int isgzip;
-	void *oi;
-	enum objiter_status ois;
-	void *sp;
-	uint8_t *pp;
-	ssize_t sl;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	p = ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, &l);
-	AN(p);
-	assert(l > 0);
-	e = p + l;
-
-	if (*p == VEC_GZ) {
-		isgzip = 1;
-		p++;
-	} else {
-		isgzip = 0;
-	}
-
-	if (req->esi_level == 0) {
-		/*
-		 * Only the top level document gets to decide this.
-		 */
-		req->gzip_resp = 0;
-		if (isgzip) {
-			assert(sizeof gzip_hdr == 10);
-			/* Send out the gzip header */
-			(void)VDP_bytes(req, VDP_NULL, gzip_hdr, 10);
-			req->l_crc = 0;
-			req->gzip_resp = 1;
-			req->crc = crc32(0L, Z_NULL, 0);
-		}
-	}
+	while (1) {
+		switch (ecx->state) {
+		case 0:
+			if (*ecx->p == VEC_GZ) {
+				ecx->isgzip = 1;
+				ecx->p++;
+			}
 
-	oi = ObjIterBegin(req->wrk, req->objcore);
-	ois = ObjIter(req->objcore, oi, &sp, &sl);
-	assert(ois != OIS_ERROR);
-	pp = sp;
-
-	while (p < e) {
-		switch (*p) {
-		case VEC_V1:
-		case VEC_V2:
-		case VEC_V8:
-			l = ved_decode_len(&p);
-			if (isgzip) {
-				assert(*p == VEC_C1 || *p == VEC_C2 ||
-				    *p == VEC_C8);
-				l_icrc = ved_decode_len(&p);
-				icrc = vbe32dec(p);
-				p += 4;
-				if (req->gzip_resp) {
-					req->crc = crc32_combine(
-					    req->crc, icrc, l_icrc);
-					req->l_crc += l_icrc;
+			if (req->esi_level == 0) {
+				/*
+				 * Only the top level document gets to
+				 * decide this.
+				 */
+				req->gzip_resp = 0;
+				if (ecx->isgzip) {
+					assert(sizeof gzip_hdr == 10);
+					/* Send out the gzip header */
+					(void)VDP_bytes(req, VDP_NULL,
+					    gzip_hdr, 10);
+					req->l_crc = 0;
+					req->gzip_resp = 1;
+					req->crc = crc32(0L, Z_NULL, 0);
 				}
 			}
-			/*
-			 * There is no guarantee that the 'l' bytes are all
-			 * in the same storage segment, so loop over storage
-			 * until we have processed them all.
-			 */
-			while (l > 0) {
-				l2 = l;
-				if (l2 > sl)
-					l2 = sl;
-				sl -= l2;
-				l -= l2;
-
-				(void)VDP_bytes(req, VDP_NULL, pp, l2);
-				pp += l2;
-				if (sl == 0) {
-					ois = ObjIter(req->objcore, oi,
-					    &sp, &sl);
-					assert(ois != OIS_ERROR);
-					pp = sp;
+			ecx->state = 1;
+
+			break;
+		case 1:
+			if (ecx->p >= ecx->e) {
+				ecx->state = 2;
+				break;
+			}
+			switch (*ecx->p) {
+			case VEC_V1:
+			case VEC_V2:
+			case VEC_V8:
+				ecx->l = ved_decode_len(&ecx->p);
+				if (ecx->isgzip) {
+					assert(*ecx->p == VEC_C1 ||
+					    *ecx->p == VEC_C2 ||
+					    *ecx->p == VEC_C8);
+					l_icrc = ved_decode_len(&ecx->p);
+					icrc = vbe32dec(ecx->p);
+					ecx->p += 4;
+					if (req->gzip_resp) {
+						req->crc = crc32_combine(
+						    req->crc, icrc, l_icrc);
+						req->l_crc += l_icrc;
+					}
+				}
+				ecx->state = 3;
+				break;
+			case VEC_S1:
+			case VEC_S2:
+			case VEC_S8:
+				ecx->l = ved_decode_len(&ecx->p);
+				Debug("SKIP1(%d)\n", (int)ecx->l);
+				ecx->state = 4;
+				break;
+			case VEC_INCL:
+				ecx->p++;
+				q = (void*)strchr((const char*)ecx->p, '\0');
+				AN(q);
+				q++;
+				r = (void*)strchr((const char*)q, '\0');
+				AN(r);
+				if (VDP_bytes(req, VDP_FLUSH, NULL, 0)) {
+					SES_Close(req->sp, SC_REM_CLOSE);
+					ecx->p = ecx->e;
+					break;
 				}
+				Debug("INCL [%s][%s] BEGIN\n", q, ecx->p);
+				ved_include(req,
+				    (const char*)q, (const char*)ecx->p);
+				Debug("INCL [%s][%s] END\n", q, ecx->p);
+				ecx->p = r + 1;
+				break;
+			default:
+				printf("XXXX 0x%02x [%s]\n", *ecx->p, ecx->p);
+				INCOMPL();
 			}
 			break;
-		case VEC_S1:
-		case VEC_S2:
-		case VEC_S8:
-			l = ved_decode_len(&p);
-			Debug("SKIP1(%d)\n", (int)l);
+		case 2:
+			if (req->gzip_resp && req->esi_level == 0) {
+				/*
+				 * We are bytealigned here, so simply emit
+				 * a gzip literal block with finish bit set.
+				 */
+				tailbuf[0] = 0x01;
+				tailbuf[1] = 0x00;
+				tailbuf[2] = 0x00;
+				tailbuf[3] = 0xff;
+				tailbuf[4] = 0xff;
+
+				/* Emit CRC32 */
+				vle32enc(tailbuf + 5, req->crc);
+
+				/* MOD(2^32) length */
+				vle32enc(tailbuf + 9, req->l_crc);
+
+				(void)VDP_bytes(req, VDP_NULL, tailbuf, 13);
+			}
+			(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
+			ecx->state = 99;
+			return;
+		case 3:
+		case 4:
 			/*
 			 * There is no guarantee that the 'l' bytes are all
 			 * in the same storage segment, so loop over storage
 			 * until we have processed them all.
 			 */
-			while (l > 0) {
-				l2 = l;
-				if (l2 > sl)
-					l2 = sl;
-				sl -= l2;
-				l -= l2;
-				pp += l2;
-				if (sl == 0) {
-					ois = ObjIter(req->objcore, oi,
-					    &sp, &sl);
-					assert(ois != OIS_ERROR);
-					pp = sp;
-				}
-			}
-			break;
-		case VEC_INCL:
-			p++;
-			q = (void*)strchr((const char*)p, '\0');
-			AN(q);
-			q++;
-			r = (void*)strchr((const char*)q, '\0');
-			AN(r);
-			if (VDP_bytes(req, VDP_FLUSH, NULL, 0)) {
-				SES_Close(req->sp, SC_REM_CLOSE);
-				p = e;
+			if (ecx->l <= sl) {
+				if (ecx->state == 3)
+					(void)VDP_bytes(req, VDP_NULL,
+					    pp, ecx->l);
+				sl -= ecx->l;
+				pp += ecx->l;
+				ecx->state = 1;
 				break;
 			}
-			Debug("INCL [%s][%s] BEGIN\n", q, p);
-			ved_include(req, (const char*)q, (const char*)p);
-			Debug("INCL [%s][%s] END\n", q, p);
-			p = r + 1;
-			break;
+			if (ecx->state == 3 && sl > 0)
+				(void)VDP_bytes(req, VDP_NULL, pp, sl);
+			ecx->l -= sl;
+			return;
+		case 99:
+			/*
+			 * VEP does not account for the PAD+CRC+LEN
+			 * so we can see up to approx 15 bytes here.
+			 */
+			return;
 		default:
-			printf("XXXX 0x%02x [%s]\n", *p, p);
-			INCOMPL();
+			WRONG("FOO");
+			break;
 		}
 	}
-	if (req->gzip_resp && req->esi_level == 0) {
-		/* Emit a gzip literal block with finish bit set */
-		tailbuf[0] = 0x01;
-		tailbuf[1] = 0x00;
-		tailbuf[2] = 0x00;
-		tailbuf[3] = 0xff;
-		tailbuf[4] = 0xff;
+}
 
-		/* Emit CRC32 */
-		vle32enc(tailbuf + 5, req->crc);
+/*---------------------------------------------------------------------
+ */
 
-		/* MOD(2^32) length */
-		vle32enc(tailbuf + 9, req->l_crc);
+void
+ESI_Deliver(struct req *req)
+{
+	struct ecx ecx;
+	ssize_t l;
+	enum objiter_status ois;
+	void *sp = NULL;
+	ssize_t sl;
+	void *oi;
+	uint8_t dummy = 0;
 
-		(void)VDP_bytes(req, VDP_NULL, tailbuf, 13);
-	}
-	(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+
+	memset(&ecx, 0, sizeof ecx);
+
+	ecx.p = ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, &l);
+	AN(ecx.p);
+	assert(l > 0);
+	ecx.e = ecx.p + l;
+
+	oi = ObjIterBegin(req->wrk, req->objcore);
+	do {
+		ois = ObjIter(req->objcore, oi, &sp, &sl);
+		assert(ois != OIS_ERROR);
+		esi_deliver(req, &ecx, sp, sl);
+	} while (ois != OIS_DONE);
+	assert(ecx.state == 99);
 	ObjIterEnd(req->objcore, &oi);
 }
 
+
 /*---------------------------------------------------------------------
  * Include an object in a gzip'ed ESI object delivery
  */
diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index 53c46de..f582b7c 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -1086,6 +1086,7 @@ VEP_Finish(struct vep_state *vep)
 		lcb = vep->cb(vep->vc, vep->cb_priv, 0, VGZ_ALIGN);
 		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
 	}
+	// NB: We don't acount for PAD+SUM+LEN in gzip'ed objects
 	(void)vep->cb(vep->vc, vep->cb_priv, 0, VGZ_FINISH);
 
 	AZ(VSB_finish(vep->vsb));



From phk at FreeBSD.org  Thu May  7 07:08:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 07 May 2015 09:08:12 +0200
Subject: [master] e76ad2d Make the meaty part of ESI_Deliver() look like a VDP
 function.
Message-ID: <E1YqFum-00014x-Vk@project.varnish-software.com>


commit e76ad2d246ea7505b65ac1ead9f4be2ce3b4984b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 6 21:48:00 2015 +0000

    Make the meaty part of ESI_Deliver() look like a VDP function.

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 92b7553..bfa82ca 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -31,6 +31,7 @@
 #include "config.h"
 
 #include <stdio.h>
+#include <stdlib.h>
 
 #include "cache.h"
 #include "cache_filter.h"
@@ -276,26 +277,51 @@ static const uint8_t gzip_hdr[] = {
 };
 
 struct ecx {
+	unsigned	magic;
+#define ECX_MAGIC	0x0b0f9163
 	uint8_t		*p;
 	uint8_t		*e;
-
 	int		state;
 	ssize_t		l;
-
 	int		isgzip;
 };
 
-static void
-esi_deliver(struct req *req, struct ecx *ecx, const uint8_t *pp, ssize_t sl)
+static int __match_proto__(vdp_bytes)
+VDP_ESI(struct req *req, enum vdp_action act, void **priv,
+    const void *ptr, ssize_t len)
 {
 	uint8_t *q, *r;
-	ssize_t l_icrc = 0;
+	ssize_t l = 0;
 	uint32_t icrc = 0;
 	uint8_t tailbuf[8 + 5];
+	const uint8_t *pp;
+	struct ecx *ecx;
+	int retval = 0;
+
+	if (act == VDP_INIT) {
+		AZ(*priv);
+		ALLOC_OBJ(ecx, ECX_MAGIC);
+		AN(ecx);
+		*priv = ecx;
+		return (0);
+	}
+	CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC);
+	if (act == VDP_FINI) {
+		FREE_OBJ(ecx);
+		*priv = NULL;
+		return (0);
+	}
+	pp = ptr;
 
 	while (1) {
 		switch (ecx->state) {
 		case 0:
+			ecx->p = ObjGetattr(req->wrk, req->objcore,
+			    OA_ESIDATA, &l);
+			AN(ecx->p);
+			assert(l > 0);
+			ecx->e = ecx->p + l;
+
 			if (*ecx->p == VEC_GZ) {
 				ecx->isgzip = 1;
 				ecx->p++;
@@ -310,7 +336,7 @@ esi_deliver(struct req *req, struct ecx *ecx, const uint8_t *pp, ssize_t sl)
 				if (ecx->isgzip) {
 					assert(sizeof gzip_hdr == 10);
 					/* Send out the gzip header */
-					(void)VDP_bytes(req, VDP_NULL,
+					retval = VDP_bytes(req, VDP_NULL,
 					    gzip_hdr, 10);
 					req->l_crc = 0;
 					req->gzip_resp = 1;
@@ -334,13 +360,13 @@ esi_deliver(struct req *req, struct ecx *ecx, const uint8_t *pp, ssize_t sl)
 					assert(*ecx->p == VEC_C1 ||
 					    *ecx->p == VEC_C2 ||
 					    *ecx->p == VEC_C8);
-					l_icrc = ved_decode_len(&ecx->p);
+					l = ved_decode_len(&ecx->p);
 					icrc = vbe32dec(ecx->p);
 					ecx->p += 4;
 					if (req->gzip_resp) {
 						req->crc = crc32_combine(
-						    req->crc, icrc, l_icrc);
-						req->l_crc += l_icrc;
+						    req->crc, icrc, l);
+						req->l_crc += l;
 					}
 				}
 				ecx->state = 3;
@@ -395,9 +421,9 @@ esi_deliver(struct req *req, struct ecx *ecx, const uint8_t *pp, ssize_t sl)
 
 				(void)VDP_bytes(req, VDP_NULL, tailbuf, 13);
 			}
-			(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
+			retval = VDP_bytes(req, VDP_FLUSH, NULL, 0);
 			ecx->state = 99;
-			return;
+			return (retval);
 		case 3:
 		case 4:
 			/*
@@ -405,29 +431,31 @@ esi_deliver(struct req *req, struct ecx *ecx, const uint8_t *pp, ssize_t sl)
 			 * in the same storage segment, so loop over storage
 			 * until we have processed them all.
 			 */
-			if (ecx->l <= sl) {
+			if (ecx->l <= len) {
 				if (ecx->state == 3)
-					(void)VDP_bytes(req, VDP_NULL,
+					retval = VDP_bytes(req, act,
 					    pp, ecx->l);
-				sl -= ecx->l;
+				len -= ecx->l;
 				pp += ecx->l;
 				ecx->state = 1;
 				break;
 			}
-			if (ecx->state == 3 && sl > 0)
-				(void)VDP_bytes(req, VDP_NULL, pp, sl);
-			ecx->l -= sl;
-			return;
+			if (ecx->state == 3 && len > 0)
+				retval = VDP_bytes(req, act, pp, len);
+			ecx->l -= len;
+			return (retval);
 		case 99:
 			/*
 			 * VEP does not account for the PAD+CRC+LEN
 			 * so we can see up to approx 15 bytes here.
 			 */
-			return;
+			return (retval);
 		default:
 			WRONG("FOO");
 			break;
 		}
+		if (retval)
+			return (retval);
 	}
 }
 
@@ -437,31 +465,23 @@ esi_deliver(struct req *req, struct ecx *ecx, const uint8_t *pp, ssize_t sl)
 void
 ESI_Deliver(struct req *req)
 {
-	struct ecx ecx;
-	ssize_t l;
 	enum objiter_status ois;
 	void *sp = NULL;
 	ssize_t sl;
 	void *oi;
-	uint8_t dummy = 0;
+	void *vp = NULL;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
-	memset(&ecx, 0, sizeof ecx);
-
-	ecx.p = ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, &l);
-	AN(ecx.p);
-	assert(l > 0);
-	ecx.e = ecx.p + l;
-
+	(void)VDP_ESI(req, VDP_INIT, &vp, NULL, 0);
 	oi = ObjIterBegin(req->wrk, req->objcore);
 	do {
 		ois = ObjIter(req->objcore, oi, &sp, &sl);
 		assert(ois != OIS_ERROR);
-		esi_deliver(req, &ecx, sp, sl);
+		(void)VDP_ESI(req, VDP_FLUSH, &vp, sp, sl);
 	} while (ois != OIS_DONE);
-	assert(ecx.state == 99);
+	(void)VDP_ESI(req, VDP_FINI, &vp, NULL, 0);
 	ObjIterEnd(req->objcore, &oi);
 }
 



From phk at FreeBSD.org  Thu May  7 07:08:13 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 07 May 2015 09:08:13 +0200
Subject: [master] 0868f6d white space ocd
Message-ID: <E1YqFun-000157-0r@project.varnish-software.com>


commit 0868f6d6234b9a297fa37cdaa07f8a0ce737dda7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 7 06:05:08 2015 +0000

    white space ocd

diff --git a/bin/varnishtest/tests/r01730.vtc b/bin/varnishtest/tests/r01730.vtc
index d19b479..2e15ead 100644
--- a/bin/varnishtest/tests/r01730.vtc
+++ b/bin/varnishtest/tests/r01730.vtc
@@ -3,7 +3,7 @@ varnishtest "Test connection error on pipe"
 varnish v1 -vcl {
        backend default { .host = "${bad_ip}"; }
        sub vcl_recv {
-               return (pipe);
+		return (pipe);
        }
 } -start
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index c541e8f..27b3e17 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -670,14 +670,14 @@ sp_variables = [
 		The corresponding HTTP header.
 		"""
 	),
-        ('resp.is_streaming',
-                'BOOL',
-                ( 'deliver', 'synth', ),
-                ( ), """
-                Returns true when the response will be streamed
-                from the backend.
-                """
-        ),
+	('resp.is_streaming',
+		'BOOL',
+		( 'deliver', 'synth', ),
+		( ), """
+		Returns true when the response will be streamed
+		from the backend.
+		"""
+	),
 	('now',
 		'TIME',
 		( 'all',),



From phk at FreeBSD.org  Thu May  7 07:08:13 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 07 May 2015 09:08:13 +0200
Subject: [master] d8d9cd0 Whitespace ocd
Message-ID: <E1YqFun-00015G-1s@project.varnish-software.com>


commit d8d9cd02baca776af7ac821be642fe90dc7ef5c7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 7 06:05:58 2015 +0000

    Whitespace ocd

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 4e40ae1..8464662 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -996,7 +996,7 @@ cmd_http_recv(CMD_ARGS)
 		i = read(hp->fd, u, n > 32 ? 32 : n);
 		if (i > 0)
 			vtc_dump(hp->vl, 4, "recv", u, i);
-		else 
+		else
 			vtc_log(hp->vl, hp->fatal, "recv() got %d (%s)", i,
 			    strerror(errno));
 		n -= i;



From phk at FreeBSD.org  Thu May  7 07:08:13 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 07 May 2015 09:08:13 +0200
Subject: [master] dc4e747 Call ESI_Deliver() as a VDP.
Message-ID: <E1YqFun-00015P-35@project.varnish-software.com>


commit dc4e7479de7d3a6f04e6cc1a80d097d748531e1d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 7 06:30:14 2015 +0000

    Call ESI_Deliver() as a VDP.
    
    Only allow VDP_ungzip to set a length if it is the top VDP.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2041a74..bad856f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1107,7 +1107,6 @@ char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
 void VRTPRIV_init(struct vrt_privs *privs);
 void VRTPRIV_dynamic_kill(struct vrt_privs *privs, uintptr_t id);
 
-void ESI_Deliver(struct req *);
 void ESI_DeliverChild(struct req *, struct busyobj *);
 
 /* cache_vrt_vmod.c */
diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c
index 86f4df2..34769c0 100644
--- a/bin/varnishd/cache/cache_deliver_proc.c
+++ b/bin/varnishd/cache/cache_deliver_proc.c
@@ -113,11 +113,6 @@ VDP_DeliverObj(struct req *req)
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	if (req->res_mode & RES_ESI) {
-		ESI_Deliver(req);
-		return (OIS_DONE);
-	}
-
 	oi = ObjIterBegin(req->wrk, req->objcore);
 	XXXAN(oi);
 	AZ(req->synth_body);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index bfa82ca..79d35d9 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -286,7 +286,7 @@ struct ecx {
 	int		isgzip;
 };
 
-static int __match_proto__(vdp_bytes)
+int __match_proto__(vdp_bytes)
 VDP_ESI(struct req *req, enum vdp_action act, void **priv,
     const void *ptr, ssize_t len)
 {
@@ -460,33 +460,6 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 }
 
 /*---------------------------------------------------------------------
- */
-
-void
-ESI_Deliver(struct req *req)
-{
-	enum objiter_status ois;
-	void *sp = NULL;
-	ssize_t sl;
-	void *oi;
-	void *vp = NULL;
-
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-
-	(void)VDP_ESI(req, VDP_INIT, &vp, NULL, 0);
-	oi = ObjIterBegin(req->wrk, req->objcore);
-	do {
-		ois = ObjIter(req->objcore, oi, &sp, &sl);
-		assert(ois != OIS_ERROR);
-		(void)VDP_ESI(req, VDP_FLUSH, &vp, sp, sl);
-	} while (ois != OIS_DONE);
-	(void)VDP_ESI(req, VDP_FINI, &vp, NULL, 0);
-	ObjIterEnd(req->objcore, &oi);
-}
-
-
-/*---------------------------------------------------------------------
  * Include an object in a gzip'ed ESI object delivery
  */
 
@@ -693,6 +666,8 @@ ESI_DeliverChild(struct req *req, struct busyobj *bo)
 		ved_stripgzip(req);
 		(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
 	} else {
+		if (req->res_mode & RES_ESI)
+			VDP_push(req, VDP_ESI, NULL, 0);
 		if (req->gzip_resp && !i)
 			VDP_push(req, ved_pretend_gzip, NULL, 0);
 		else if (!req->gzip_resp && i)
diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h
index bbc3f8e..42f9c91 100644
--- a/bin/varnishd/cache/cache_filter.h
+++ b/bin/varnishd/cache/cache_filter.h
@@ -111,3 +111,4 @@ void VDP_close(struct req *req);
 enum objiter_status VDP_DeliverObj(struct req *req);
 
 vdp_bytes VDP_gunzip;
+vdp_bytes VDP_ESI;
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 041d890..3bc1ed0 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -296,8 +296,12 @@ VDP_gunzip(struct req *req, enum vdp_action act, void **priv,
 		p = ObjGetattr(req->wrk, req->objcore, OA_GZIPBITS, &dl);
 		if (p != NULL && dl == 32) {
 			u = vbe64dec(p + 24);
-			/* XXX: Zero is suspect: OA_GZIPBITS wasn't set */
-			if (u != 0)
+			/*
+			 * If the size is non-zero, and we are the top
+			 * VDP, we know what size the output will be.
+			 */
+			if (u != 0 &&
+			    VTAILQ_FIRST(&req->vdp)->func == VDP_gunzip)
 				req->resp_len = u;
 			else
 				req->resp_len = -1;
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index 27d4f72..c2cc57c 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -90,6 +90,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 
 	if (req->res_mode & RES_ESI) {
 		RFC2616_Weaken_Etag(req->resp);
+		VDP_push(req, VDP_ESI, NULL, 0);
 	} else if (http_IsStatus(req->resp, 304)) {
 		http_Unset(req->resp, H_Content_Length);
 		req->wantbody = 0;
@@ -105,13 +106,13 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 		 * XXX: with multiple writes because of the gunzip buffer
 		 */
 		req->res_mode |= RES_GUNZIP;
-		VDP_push(req, VDP_gunzip, NULL, 0);
+		VDP_push(req, VDP_gunzip, NULL, 1);
 	}
 
 	if (req->res_mode & RES_ESI) {
 		/* Gunzip could have added back a C-L */
 		http_Unset(req->resp, H_Content_Length);
-		req->resp_len = -1;
+		assert(req->resp_len < 0);
 	}
 
 	/*



From phk at FreeBSD.org  Thu May  7 09:12:57 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 07 May 2015 11:12:57 +0200
Subject: [master] 092ab4d Get closer to req->resp_len meaning "bytes to
 actually send, if known"
Message-ID: <E1YqHrV-0000rq-JQ@project.varnish-software.com>


commit 092ab4d952a8d88e2c665cf43d57a8fae112d54a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 7 09:12:09 2015 +0000

    Get closer to req->resp_len meaning "bytes to actually send, if known"

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bad856f..71dc7b0 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -985,7 +985,7 @@ size_t V1L_Write(const struct worker *w, const void *ptr, ssize_t len);
 task_func_t VPX_Proto_Sess;
 
 /* cache_range.c [VRG] */
-void VRG_dorange(struct req *req, const struct busyobj *bo, const char *r);
+void VRG_dorange(struct req *req, const char *r);
 
 /* cache_req.c */
 struct req *Req_New(const struct worker *, struct sess *);
diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index fcff78a..55a78ba 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -86,7 +86,7 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv,
 /*--------------------------------------------------------------------*/
 
 static int
-vrg_dorange(struct req *req, ssize_t len, const char *r)
+vrg_dorange(struct req *req, const char *r)
 {
 	ssize_t low, high, has_low, has_high, t;
 	struct vrg_priv *vrg_priv;
@@ -127,16 +127,16 @@ vrg_dorange(struct req *req, ssize_t len, const char *r)
 		return (__LINE__);
 
 	if (!has_low) {
-		if (len < 0)
+		if (req->resp_len < 0)
 			return (0);		// Allow 200 response
 		if (high == 0)
 			return (__LINE__);
-		low = len - high;
+		low = req->resp_len - high;
 		if (low < 0)
 			low = 0;
-		high = len - 1;
-	} else if (len >= 0 && (high >= len || !has_high))
-		high = len - 1;
+		high = req->resp_len - 1;
+	} else if (req->resp_len >= 0 && (high >= req->resp_len || !has_high))
+		high = req->resp_len - 1;
 	else if (!has_high)
 		return (0);			// Allow 200 response
 	/*
@@ -149,12 +149,12 @@ vrg_dorange(struct req *req, ssize_t len, const char *r)
 	if (high < low)
 		return (__LINE__);
 
-	if (len >= 0 && low >= len)
+	if (req->resp_len >= 0 && low >= req->resp_len)
 		return (__LINE__);
 
-	if (len >= 0)
+	if (req->resp_len >= 0)
 		http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd",
-		    (intmax_t)low, (intmax_t)high, (intmax_t)len);
+		    (intmax_t)low, (intmax_t)high, (intmax_t)req->resp_len);
 	else
 		http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/*",
 		    (intmax_t)low, (intmax_t)high);
@@ -172,9 +172,8 @@ vrg_dorange(struct req *req, ssize_t len, const char *r)
 }
 
 void
-VRG_dorange(struct req *req, const struct busyobj *bo, const char *r)
+VRG_dorange(struct req *req, const char *r)
 {
-	ssize_t len;
 	int i;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -182,19 +181,17 @@ VRG_dorange(struct req *req, const struct busyobj *bo, const char *r)
 	assert(http_IsStatus(req->resp, 200));
 
 	/* We must snapshot the length if we're streaming from the backend */
-	if (bo != NULL)
-		len = http_GetContentLength(bo->beresp);
-	else
-		len = ObjGetLen(req->wrk, req->objcore);
 
-	i = vrg_dorange(req, len, r);
+	i = vrg_dorange(req, r);
 	if (i) {
 		VSLb(req->vsl, SLT_Debug, "RANGE_FAIL line %d", i);
 		http_Unset(req->resp, H_Content_Length);
-		if (bo == NULL)
+		if (req->resp_len >= 0)
 			http_PrintfHeader(req->resp,
-			    "Content-Range: bytes */%jd", (intmax_t)len);
+			    "Content-Range: bytes */%jd",
+			    (intmax_t)req->resp_len);
 		http_PutResponse(req->resp, "HTTP/1.1", 416, NULL);
+		req->resp_len = -1;
 		req->wantbody = 0;
 	}
 }
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index c2cc57c..e541f01 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -70,7 +70,10 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
 	req->res_mode = 0;
-	req->resp_len = -2;
+	if (bo != NULL)
+		req->resp_len = http_GetContentLength(bo->beresp);
+	else
+		req->resp_len = ObjGetLen(req->wrk, req->objcore);
 
 	/*
 	 * Determine ESI status first.  Not dependent on wantbody, because
@@ -90,6 +93,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 
 	if (req->res_mode & RES_ESI) {
 		RFC2616_Weaken_Etag(req->resp);
+		req->resp_len = -1;
 		VDP_push(req, VDP_ESI, NULL, 0);
 	} else if (http_IsStatus(req->resp, 304)) {
 		http_Unset(req->resp, H_Content_Length);
@@ -109,11 +113,8 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 		VDP_push(req, VDP_gunzip, NULL, 1);
 	}
 
-	if (req->res_mode & RES_ESI) {
-		/* Gunzip could have added back a C-L */
-		http_Unset(req->resp, H_Content_Length);
+	if (req->res_mode & RES_ESI)
 		assert(req->resp_len < 0);
-	}
 
 	/*
 	 * Range comes after the others and pushes on bottom because it
@@ -122,17 +123,22 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 	if (cache_param->http_range_support && http_IsStatus(req->resp, 200)) {
 		http_SetHeader(req->resp, "Accept-Ranges: bytes");
 		if (req->wantbody && http_GetHdr(req->http, H_Range, &r))
-			VRG_dorange(req, bo, r);
+			VRG_dorange(req, r);
 	}
 
-	if (req->resp_len >= -1 && req->wantbody)
+	if ((req->objcore->flags & OC_F_PRIVATE) &&
+	    !strcasecmp(http_GetMethod(req->http0), "HEAD")) {
+		/* HEAD+pass is allowed to send the C-L through unmolested. */
+	} else {
 		http_Unset(req->resp, H_Content_Length);
-	if (req->resp_len >= 0 && req->wantbody)
-		http_PrintfHeader(req->resp,
-		    "Content-Length: %jd", req->resp_len);
+		if (req->resp_len >= 0 && !http_IsStatus(req->resp, 304)) 
+			http_PrintfHeader(req->resp,
+			    "Content-Length: %jd", req->resp_len);
+	}
 
 	if (http_GetHdr(req->resp, H_Content_Length, NULL))
 		req->res_mode |= RES_LEN;
+
 	else if (req->wantbody) {
 		if (req->http->protover == 11) {
 			req->res_mode |= RES_CHUNKED;



From phk at FreeBSD.org  Thu May  7 11:17:57 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 07 May 2015 13:17:57 +0200
Subject: [master] 7333dde Pull the protocol-agnostic bits of VDP (esi, range,
 gunzip) out of V1D and plunk it down in req_fsm for now.
Message-ID: <E1YqJoT-0005KA-Ni@project.varnish-software.com>


commit 7333dde2148c6602f096bbeedffdf03212af0cbc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 7 11:17:12 2015 +0000

    Pull the protocol-agnostic bits of VDP (esi, range, gunzip) out of V1D
    and plunk it down in req_fsm for now.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 71dc7b0..7c6a076 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -741,7 +741,7 @@ extern const int HTTP1_Req[3];
 extern const int HTTP1_Resp[3];
 
 /* cache_http1_deliver.c */
-void V1D_Deliver(struct req *, struct busyobj *);
+void V1D_Deliver(struct req *);
 
 /* cache_http1_pipe.c */
 void V1P_Init(void);
@@ -1107,7 +1107,7 @@ char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
 void VRTPRIV_init(struct vrt_privs *privs);
 void VRTPRIV_dynamic_kill(struct vrt_privs *privs, uintptr_t id);
 
-void ESI_DeliverChild(struct req *, struct busyobj *);
+int VED_Setup(struct req *req, struct busyobj *bo);
 
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 79d35d9..7f072ea 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -653,11 +653,30 @@ ved_stripgzip(struct req *req)
 	req->l_crc += ilen;
 }
 
-void
-ESI_DeliverChild(struct req *req, struct busyobj *bo)
+int
+VED_Setup(struct req *req, struct busyobj *bo)
 {
 	int i;
 
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+
+	/*
+	 * Determine ESI status first.  Not dependent on wantbody, because
+	 * we want ESI to supress C-L in HEAD too.
+	 */
+	if (!req->disable_esi &&
+	    ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, NULL) != NULL) {
+		req->res_mode |= RES_ESI;
+		RFC2616_Weaken_Etag(req->resp);
+		req->resp_len = -1;
+		VDP_push(req, VDP_ESI, NULL, 0);
+	}
+
+	/* ESI-childen need special treatment */
+	if (req->esi_level == 0)
+		return (0);
+
 	req->res_mode |= RES_ESI_CHILD;
 	i = ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED);
 	if (req->gzip_resp && i && !(req->res_mode & RES_ESI)) {
@@ -666,8 +685,6 @@ ESI_DeliverChild(struct req *req, struct busyobj *bo)
 		ved_stripgzip(req);
 		(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
 	} else {
-		if (req->res_mode & RES_ESI)
-			VDP_push(req, VDP_ESI, NULL, 0);
 		if (req->gzip_resp && !i)
 			VDP_push(req, ved_pretend_gzip, NULL, 0);
 		else if (!req->gzip_resp && i)
@@ -676,4 +693,5 @@ ESI_DeliverChild(struct req *req, struct busyobj *bo)
 		(void)VDP_DeliverObj(req);
 	}
 	VDP_close(req);
+	return (1);
 }
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 42e67c3..590b5d7 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -40,12 +40,49 @@
 
 #include "cache.h"
 #include "cache_director.h"
+#include "cache_filter.h"
 
 #include "hash/hash_slinger.h"
 #include "vcl.h"
 #include "vsha256.h"
 #include "vtim.h"
 
+static void
+cnt_vdp(struct req *req, struct busyobj *bo)
+{
+	const char *r;
+
+	req->res_mode = 0;
+	if (bo != NULL)
+		req->resp_len = http_GetContentLength(bo->beresp);
+	else
+		req->resp_len = ObjGetLen(req->wrk, req->objcore);
+
+	if (VED_Setup(req, bo))
+		return;
+
+	if (cache_param->http_gzip_support &&
+	    ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
+	    !RFC2616_Req_Gzip(req->http)) {
+		req->res_mode |= RES_GUNZIP;
+		VDP_push(req, VDP_gunzip, NULL, 1);
+	}
+
+	/*
+	 * Range comes after the others and pushes on bottom because
+	 * it can generate a correct C-L header.
+	 */
+	if (cache_param->http_range_support &&
+	    http_IsStatus(req->resp, 200)) {
+		http_SetHeader(req->resp, "Accept-Ranges: bytes");
+		if (req->wantbody &&
+		    http_GetHdr(req->http, H_Range, &r))
+			VRG_dorange(req, r);
+	}
+
+	V1D_Deliver(req);
+}
+
 /*--------------------------------------------------------------------
  * Deliver an object to client
  */
@@ -144,7 +181,9 @@ cnt_deliver(struct worker *wrk, struct req *req)
 			VBO_DerefBusyObj(wrk, &bo);
 		}
 	}
-	V1D_Deliver(req, bo);
+
+	cnt_vdp(req, bo);
+
 	if (bo != NULL)
 		VBO_DerefBusyObj(wrk, &bo);
 
@@ -235,7 +274,7 @@ cnt_synth(struct worker *wrk, struct req *req)
 		VSB_delete(req->synth_body);
 		req->synth_body = NULL;
 
-		V1D_Deliver(req, NULL);
+		cnt_vdp(req, NULL);
 		(void)HSH_DerefObjCore(wrk, &req->objcore);
 	}
 
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index 19857c1..984a3f1 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -61,67 +61,13 @@ v1d_bytes(struct req *req, enum vdp_action act, void **priv,
  */
 
 void
-V1D_Deliver(struct req *req, struct busyobj *bo)
+V1D_Deliver(struct req *req)
 {
-	const char *r;
 	enum objiter_status ois;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
-	req->res_mode = 0;
-	if (bo != NULL)
-		req->resp_len = http_GetContentLength(bo->beresp);
-	else
-		req->resp_len = ObjGetLen(req->wrk, req->objcore);
-
-	/*
-	 * Determine ESI status first.  Not dependent on wantbody, because
-	 * we want ESI to supress C-L in HEAD too.
-	 */
-	if (!req->disable_esi &&
-	    ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, NULL) != NULL)
-		req->res_mode |= RES_ESI;
-
-	/*
-	 * ESI-childen don't care about headers -> early escape
-	 */
-	if (req->esi_level > 0) {
-		ESI_DeliverChild(req, bo);
-		return;
-	}
-
-	if (req->res_mode & RES_ESI) {
-		RFC2616_Weaken_Etag(req->resp);
-		req->resp_len = -1;
-		VDP_push(req, VDP_ESI, NULL, 0);
-	}
-
-	if (cache_param->http_gzip_support &&
-	    ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
-	    !RFC2616_Req_Gzip(req->http)) {
-		/*
-		 * We don't know what it uncompresses to
-		 * XXX: we could cache that, but would still deliver
-		 * XXX: with multiple writes because of the gunzip buffer
-		 */
-		req->res_mode |= RES_GUNZIP;
-		VDP_push(req, VDP_gunzip, NULL, 1);
-	}
-
-	if (req->res_mode & RES_ESI)
-		assert(req->resp_len < 0);
-
-	/*
-	 * Range comes after the others and pushes on bottom because it
-	 * can generate a correct C-L header.
-	 */
-	if (cache_param->http_range_support && http_IsStatus(req->resp, 200)) {
-		http_SetHeader(req->resp, "Accept-Ranges: bytes");
-		if (req->wantbody && http_GetHdr(req->http, H_Range, &r))
-			VRG_dorange(req, r);
-	}
-
 	if ((req->objcore->flags & OC_F_PRIVATE) &&
 	    !strcasecmp(http_GetMethod(req->http0), "HEAD")) {
 		/* HEAD+pass is allowed to send the C-L through unmolested. */



From phk at FreeBSD.org  Thu May  7 11:17:57 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 07 May 2015 13:17:57 +0200
Subject: [master] b86ffb9 Move 304/C-L/Body logic earlier, making it simpler
Message-ID: <E1YqJoT-0005K4-MM@project.varnish-software.com>


commit b86ffb93487c107f8f92917bfd15c7a0223f42ed
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 7 09:45:38 2015 +0000

    Move 304/C-L/Body logic earlier, making it simpler

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index d8ade2d..42e67c3 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -131,6 +131,9 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		req->wantbody = 0;
 	}
 
+	if (http_IsStatus(req->resp, 304))
+		req->wantbody = 0;
+
 	/* Grab a ref to the bo if there is one, and hand it down */
 	bo = HSH_RefBusy(req->objcore);
 	if (bo != NULL) {
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index e541f01..19857c1 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -95,11 +95,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 		RFC2616_Weaken_Etag(req->resp);
 		req->resp_len = -1;
 		VDP_push(req, VDP_ESI, NULL, 0);
-	} else if (http_IsStatus(req->resp, 304)) {
-		http_Unset(req->resp, H_Content_Length);
-		req->wantbody = 0;
-	} else if (bo == NULL && req->wantbody)
-		req->resp_len = ObjGetLen(req->wrk, req->objcore);
+	}
 
 	if (cache_param->http_gzip_support &&
 	    ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
@@ -131,7 +127,7 @@ V1D_Deliver(struct req *req, struct busyobj *bo)
 		/* HEAD+pass is allowed to send the C-L through unmolested. */
 	} else {
 		http_Unset(req->resp, H_Content_Length);
-		if (req->resp_len >= 0 && !http_IsStatus(req->resp, 304)) 
+		if (req->resp_len >= 0 && !http_IsStatus(req->resp, 304))
 			http_PrintfHeader(req->resp,
 			    "Content-Length: %jd", req->resp_len);
 	}



From phk at FreeBSD.org  Mon May 11 08:46:41 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 11 May 2015 10:46:41 +0200
Subject: [master] 8c0af33 Fix an attempt to double close a session.
Message-ID: <E1YrjMH-0008ND-Cy@project.varnish-software.com>


commit 8c0af33385c8045cea0574af26418f5480c13e50
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 11 08:45:22 2015 +0000

    Fix an attempt to double close a session.
    
    In future versions of HTTP requests can be terminated abnormally
    without ditching the entire session, and this change starts the
    transition to that logic.
    
    Fixes: #1732

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7c6a076..29aeaec 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -991,6 +991,7 @@ void VRG_dorange(struct req *req, const char *r);
 struct req *Req_New(const struct worker *, struct sess *);
 void Req_Release(struct req *);
 int Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req);
+void Req_Fail(struct req *req, enum sess_close reason);
 
 /* cache_session.c [SES] */
 struct sess *SES_New(struct sesspool *);
diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index 55a78ba..23c9925 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -59,7 +59,7 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv,
 	CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC);
 	if (act == VDP_FINI) {
 		if (vrg_priv->range_off < vrg_priv->range_high)
-			SES_Close(req->sp, SC_RANGE_SHORT);
+			Req_Fail(req, SC_RANGE_SHORT);
 		*priv = NULL;	/* struct on ws, no need to free */
 		return (0);
 	}
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 1804b1c..03c3557 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -203,3 +203,15 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Reset(wrk->aws, NULL);
 	return (0);
 }
+
+/*----------------------------------------------------------------------
+ */
+
+void __match_proto__()
+Req_Fail(struct req *req, enum sess_close reason)
+{
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+
+	if (req->sp->fd >= 0)
+		SES_Close(req->sp, reason);
+}
diff --git a/bin/varnishtest/tests/r01732.vtc b/bin/varnishtest/tests/r01732.vtc
new file mode 100644
index 0000000..f738dec
--- /dev/null
+++ b/bin/varnishtest/tests/r01732.vtc
@@ -0,0 +1,38 @@
+varnishtest "range related panic"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	chunkedlen 10
+	sema r1 sync 2
+	chunkedlen 10
+	chunkedlen 10
+	chunkedlen 10
+	chunkedlen 0
+	delay .1
+	sema r2 sync 2
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	txreq -hdr "Range: bytes=0-100"
+	rxresphdrs
+	expect resp.status == 206
+	expect resp.http.Content-Range == "bytes 0-100/*"
+} -run
+
+delay .1
+sema r1 sync 2
+sema r2 sync 2
+delay .4
+
+client c1 {
+	txreq -hdr "Range: bytes=0-100"
+	rxresp
+	expect resp.status == 206
+	expect resp.http.Content-Range == "bytes 0-49/50"
+} -run
+



From phk at FreeBSD.org  Mon May 11 09:10:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 11 May 2015 11:10:12 +0200
Subject: [master] d0012bd Retire the backend_toolate counter.
Message-ID: <E1Yrjj2-0002VG-Tt@project.varnish-software.com>


commit d0012bd98c55446624a47cb256c28ff4afc01809
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 11 09:09:31 2015 +0000

    Retire the backend_toolate counter.
    
    With backend waiters it makes no sense to bring it back.
    
    Fixes: #1726

diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index fc7d783..3c6c8d0 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -130,10 +130,6 @@ VSC_F(backend_reuse,		uint64_t, 0, 'c', 'i', info,
 	"Count of backend connection reuses"
 	"  This counter is increased whenever we reuse a recycled connection."
 )
-VSC_F(backend_toolate,		uint64_t, 0, 'c', 'i', info,
-    "Backend conn. was closed",
-	""
-)
 VSC_F(backend_recycle,		uint64_t, 0, 'c', 'i', info,
     "Backend conn. recycles",
 	"Count of backend connection recycles"



From phk at FreeBSD.org  Wed May 13 08:16:19 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 13 May 2015 10:16:19 +0200
Subject: [master] d573f35 Only close backend connection on retry if it has a
 body
Message-ID: <E1YsRpz-0002HI-Vf@project.varnish-software.com>


commit d573f35f63d40f07c1bf4bb4c27ab7c2e0e68176
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 11 10:26:48 2015 +0000

    Only close backend connection on retry if it has a body

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 38703fe..c36722c 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -286,6 +286,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	assert(bo->state <= BOS_REQ_DONE);
 
+	AZ(bo->htc);
 	i = VDI_GetHdr(wrk, bo);
 
 	now = W_TIM_real(wrk);
@@ -434,7 +435,8 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	}
 
 	if (wrk->handling == VCL_RET_RETRY) {
-		bo->doclose = SC_RESP_CLOSE;
+		if (bo->htc->body_status != BS_NONE)
+			bo->doclose = SC_RESP_CLOSE;
 		if (bo->director_state != DIR_S_NULL)
 			VDI_Finish(bo->wrk, bo);
 



From phk at FreeBSD.org  Wed May 13 08:16:20 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 13 May 2015 10:16:20 +0200
Subject: [master] 735523e Fail if backend workspace doesn't have room for htc
Message-ID: <E1YsRq0-0002HO-0q@project.varnish-software.com>


commit 735523ef6f84b24e265570d633def0ae16ae10e8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 11 10:27:08 2015 +0000

    Fail if backend workspace doesn't have room for htc

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index c555e05..32189df 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -129,9 +129,11 @@ vbe_dir_getfd(const struct director *d, struct busyobj *bo)
 	    vc->fd, bp->display_name, abuf2, pbuf2, abuf1, pbuf1);
 
 	vc->backend->vsc->req++;
+	AZ(bo->htc);
+	bo->htc = WS_Alloc(bo->ws, sizeof *bo->htc);
 	if (bo->htc == NULL)
-		bo->htc = WS_Alloc(bo->ws, sizeof *bo->htc);
-	AN(bo->htc);
+		/* XXX: counter ? */
+		return (-1);
 	INIT_OBJ(bo->htc, HTTP_CONN_MAGIC);
 	bo->htc->vbc = vc;
 	bo->htc->fd = vc->fd;



From phk at FreeBSD.org  Wed May 13 08:16:20 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 13 May 2015 10:16:20 +0200
Subject: [master] ade0db0 Stop the epoll_waiter from holding on to fd's while
 stuff happens,
 it is supected of being the cause of the race in #1675 amongst other evils.
Message-ID: <E1YsRq0-0002HX-27@project.varnish-software.com>


commit ade0db0b9394449399332686bca47cc1c8e9f51c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 13 08:05:52 2015 +0000

    Stop the epoll_waiter from holding on to fd's while stuff happens,
    it is supected of being the cause of the race in #1675 amongst other
    evils.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 29aeaec..565a4bb 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -52,10 +52,6 @@
 #include <unistd.h>
 #include <math.h>
 
-#if defined(HAVE_EPOLL_CTL)
-#include <sys/epoll.h>
-#endif
-
 #include "common/params.h"
 
 /*--------------------------------------------------------------------*/
@@ -381,9 +377,6 @@ struct waited {
 	VTAILQ_ENTRY(waited)	list;
 	void			*ptr;
 	double			idle;
-#if defined(HAVE_EPOLL_CTL)
-	struct epoll_event ev;
-#endif
 };
 
 /* Stored object -----------------------------------------------------
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index 56987cd..f1e8bd9 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -65,37 +65,34 @@ static void
 vwe_inject(const struct waiter *w, struct waited *wp)
 {
 	struct vwe *vwe;
+	struct epoll_event ev;
 
 	CAST_OBJ_NOTNULL(vwe, w->priv, VWE_MAGIC);
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 	assert(wp->fd >= 0);
 
-	if (wp->ev.data.ptr)
-		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_MOD, wp->fd, &wp->ev));
-	else {
-		wp->ev.data.ptr = wp;
-		wp->ev.events = EPOLLIN | EPOLLRDHUP;
-		if (wp != w->pipe_w)
-			wp->ev.events |= EPOLLONESHOT;
-		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_ADD, wp->fd, &wp->ev));
-	}
+	ev.data.ptr = wp;
+	ev.events = EPOLLIN | EPOLLRDHUP;
+	AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_ADD, wp->fd, &ev));
 }
 
 static void
 vwe_eev(struct vwe *vwe, const struct epoll_event *ep, double now)
 {
-	struct waited *sp;
+	struct waited *wp;
 
 	AN(ep->data.ptr);
-	CAST_OBJ_NOTNULL(sp, ep->data.ptr, WAITED_MAGIC);
+	CAST_OBJ_NOTNULL(wp, ep->data.ptr, WAITED_MAGIC);
+	if (wp != vwe->waiter->pipe_w)
+		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL));
 	if (ep->events & EPOLLIN) {
-		Wait_Handle(vwe->waiter, sp, WAITER_ACTION, now);
+		Wait_Handle(vwe->waiter, wp, WAITER_ACTION, now);
 	} else if (ep->events & EPOLLERR) {
-		Wait_Handle(vwe->waiter, sp, WAITER_REMCLOSE, now);
+		Wait_Handle(vwe->waiter, wp, WAITER_REMCLOSE, now);
 	} else if (ep->events & EPOLLHUP) {
-		Wait_Handle(vwe->waiter, sp, WAITER_REMCLOSE, now);
+		Wait_Handle(vwe->waiter, wp, WAITER_REMCLOSE, now);
 	} else if (ep->events & EPOLLRDHUP) {
-		Wait_Handle(vwe->waiter, sp, WAITER_REMCLOSE, now);
+		Wait_Handle(vwe->waiter, wp, WAITER_REMCLOSE, now);
 	}
 }
 



From phk at FreeBSD.org  Wed May 13 20:37:03 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 13 May 2015 22:37:03 +0200
Subject: [master] 7f71fe6 Avoid a length=0 panic where the length can actually
 be zero.
Message-ID: <E1YsdOp-00044I-LL@project.varnish-software.com>


commit 7f71fe6e0182cff8aaa5470ffff6c74b67ad8bf7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 13 20:35:43 2015 +0000

    Avoid a length=0 panic where the length can actually be zero.
    
    Fixes #1692
    
    Found and diagnosed by:	martin
    Slightly different patch by me.

diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index f582b7c..aa31aa4 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -295,7 +295,9 @@ static void
 vep_emit_common(struct vep_state *vep, ssize_t l, enum vep_mark mark)
 {
 
-	assert(l > 0);
+	assert(l >= 0);
+	if (l == 0)
+		return;
 	assert(mark == SKIP || mark == VERBATIM);
 	if (mark == SKIP)
 		vep_emit_skip(vep, l);
@@ -330,8 +332,7 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 	if (vep->last_mark != mark && (vep->o_wait > 0 || vep->startup)) {
 		lcb = vep->cb(vep->vc, vep->cb_priv, 0,
 		    mark == VERBATIM ? VGZ_RESET : VGZ_ALIGN);
-		if (lcb - vep->o_last > 0)
-			vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
+		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
 		vep->o_last = lcb;
 		vep->o_wait = 0;
 	}



From phk at FreeBSD.org  Thu May 14 08:04:15 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 10:04:15 +0200
Subject: [master] 4e5107d Attempt to allocate the htc before the backend
 connection,
 its cheaper to eliminate the exceedingly unlikely resource leak Coverity
 spotted that way.
Message-ID: <E1Yso7r-0001vI-Uu@project.varnish-software.com>


commit 4e5107db50d41ed74fb23e316f5fb6d406b94ed4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 08:02:04 2015 +0000

    Attempt to allocate the htc before the backend connection,
    its cheaper to eliminate the exceedingly unlikely resource leak
    Coverity spotted that way.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 32189df..9aac126 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -105,11 +105,18 @@ vbe_dir_getfd(const struct director *d, struct busyobj *bo)
 		return (-1);
 	}
 
+	AZ(bo->htc);
+	bo->htc = WS_Alloc(bo->ws, sizeof *bo->htc);
+	if (bo->htc == NULL)
+		/* XXX: counter ? */
+		return (-1);
+
 	FIND_TMO(connect_timeout, tmod, bo, vrt);
 	vc = VBT_Get(bp->tcp_pool, tmod);
 	if (vc == NULL) {
 		// XXX: Per backend stats ?
 		VSC_C_main->backend_fail++;
+		bo->htc = NULL;
 		return (-1);
 	}
 
@@ -129,11 +136,6 @@ vbe_dir_getfd(const struct director *d, struct busyobj *bo)
 	    vc->fd, bp->display_name, abuf2, pbuf2, abuf1, pbuf1);
 
 	vc->backend->vsc->req++;
-	AZ(bo->htc);
-	bo->htc = WS_Alloc(bo->ws, sizeof *bo->htc);
-	if (bo->htc == NULL)
-		/* XXX: counter ? */
-		return (-1);
 	INIT_OBJ(bo->htc, HTTP_CONN_MAGIC);
 	bo->htc->vbc = vc;
 	bo->htc->fd = vc->fd;



From phk at FreeBSD.org  Thu May 14 09:43:27 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 11:43:27 +0200
Subject: [master] f1501b5 try to get more comprehensive core dumps on test
 timeouts
Message-ID: <E1Yspfr-0003fX-Ow@project.varnish-software.com>


commit f1501b5b1f14b54bff71eb3ec9ea3691a3de04c0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 08:39:06 2015 +0000

    try to get more comprehensive core dumps on test timeouts

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 6a1da82..a2b34e0 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -149,13 +149,10 @@ tst_cb(const struct vev *ve, int what)
 	CAST_OBJ_NOTNULL(jp, ve->priv, JOB_MAGIC);
 
 	// printf("%p %s %d\n", ve, jp->tst->filename, what);
-	if (what == 0) {
-		/* XXX: Timeout */
-		AZ(kill(jp->child, SIGKILL));
-		jp->evt = NULL;
-		return (1);
-	}
-	assert(what & (EV_RD | EV_HUP));
+	if (what == 0)
+		AZ(kill(jp->child, SIGKILL)); /* XXX: Timeout */
+	else
+		assert(what & (EV_RD | EV_HUP));
 
 	*buf = '\0';
 	i = read(ve->fd, buf, sizeof buf - 1);



From phk at FreeBSD.org  Thu May 14 09:43:27 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 11:43:27 +0200
Subject: [master] 303f560 Eliminate ws argument from VCL_bla_method()
 functions.
Message-ID: <E1Yspfr-0003fe-QC@project.varnish-software.com>


commit 303f56092cb010ffa9596a11de77771d4ad6810d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 08:53:45 2015 +0000

    Eliminate ws argument from VCL_bla_method() functions.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 565a4bb..e51fbb4 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1086,7 +1086,7 @@ const char *VCL_Method_Name(unsigned);
 
 #define VCL_MET_MAC(l,u,b) \
     void VCL_##l##_method(struct VCL_conf *, struct worker *, struct req *, \
-	struct busyobj *bo, struct ws *);
+	struct busyobj *bo);
 #include "tbl/vcl_returns.h"
 #undef VCL_MET_MAC
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c36722c..24df6f6 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -274,7 +274,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
 
-	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
+	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo);
 
 	bo->uncacheable = bo->do_pass;
 	if (wrk->handling == VCL_RET_ABANDON)
@@ -426,7 +426,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	bo->vfc->http = bo->beresp;
 	bo->vfc->esi_req = bo->bereq;
 
-	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, bo->beresp->ws);
+	VCL_backend_response_method(bo->vcl, wrk, NULL, bo);
 
 	if (wrk->handling == VCL_RET_ABANDON) {
 		bo->doclose = SC_RESP_CLOSE;
@@ -796,7 +796,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	bo->fetch_objcore->exp.grace = 0;
 	bo->fetch_objcore->exp.keep = 0;
 
-	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
+	VCL_backend_error_method(bo->vcl, wrk, NULL, bo);
 
 	AZ(VSB_finish(bo->synth_body));
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 590b5d7..8e48880 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -133,7 +133,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	    !RFC2616_Req_Gzip(req->http))
 		RFC2616_Weaken_Etag(req->resp);
 
-	VCL_deliver_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_deliver_method(req->vcl, wrk, req, NULL);
 	VSLb_ts_req(req, "Process", W_TIM_real(wrk));
 
 	/* Stop the insanity before it turns "Hotel California" on us */
@@ -235,7 +235,7 @@ cnt_synth(struct worker *wrk, struct req *req)
 	req->synth_body = VSB_new_auto();
 	AN(req->synth_body);
 
-	VCL_synth_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_synth_method(req->vcl, wrk, req, NULL);
 
 	http_Unset(h, H_Content_Length);
 
@@ -388,7 +388,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 
 	VSLb(req->vsl, SLT_Hit, "%u", ObjGetXID(wrk, req->objcore));
 
-	VCL_hit_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_hit_method(req->vcl, wrk, req, NULL);
 
 	switch (wrk->handling) {
 	case VCL_RET_DELIVER:
@@ -463,7 +463,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
-	VCL_miss_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_miss_method(req->vcl, wrk, req, NULL);
 	switch (wrk->handling) {
 	case VCL_RET_FETCH:
 		wrk->stats->cache_miss++;
@@ -504,7 +504,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(req->objcore);
 
-	VCL_pass_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_pass_method(req->vcl, wrk, req, NULL);
 	switch (wrk->handling) {
 	case VCL_RET_SYNTH:
 		req->req_step = R_STP_SYNTH;
@@ -550,7 +550,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(req->vsl->wid));
 	http_SetHeader(bo->bereq, "Connection: close");
 
-	VCL_pipe_method(req->vcl, wrk, req, bo, req->http->ws);
+	VCL_pipe_method(req->vcl, wrk, req, bo);
 
 	if (wrk->handling == VCL_RET_SYNTH)
 		INCOMPL();
@@ -651,7 +651,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 
 	http_CollectHdr(req->http, H_Cache_Control);
 
-	VCL_recv_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_recv_method(req->vcl, wrk, req, NULL);
 
 	/* Attempts to cache req.body may fail */
 	if (req->req_body_status == REQ_BODY_FAIL) {
@@ -674,7 +674,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 
 	req->sha256ctx = &sha256ctx;	/* so HSH_AddString() can find it */
 	SHA256_Init(req->sha256ctx);
-	VCL_hash_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_hash_method(req->vcl, wrk, req, NULL);
 	assert(wrk->handling == VCL_RET_LOOKUP);
 	SHA256_Final(req->digest, req->sha256ctx);
 	req->sha256ctx = NULL;
@@ -743,7 +743,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 
 	AZ(HSH_DerefObjCore(wrk, &boc));
 
-	VCL_purge_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VCL_purge_method(req->vcl, wrk, req, NULL);
 	switch (wrk->handling) {
 	case VCL_RET_RESTART:
 		req->req_step = R_STP_RESTART;
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index a5a7da4..ac051d1 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -461,7 +461,7 @@ ccf_config_show(struct cli *cli, const char * const *av, void *priv)
 
 static void
 vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
-    struct ws *ws, unsigned method, vcl_func_f *func)
+    unsigned method, vcl_func_f *func)
 {
 	char *aws;
 	struct vsl_log *vsl = NULL;
@@ -478,6 +478,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 		ctx.http_resp = req->resp;
 		ctx.req = req;
 		ctx.now = req->t_prev;
+		ctx.ws = req->ws;
 	}
 	if (bo != NULL) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -487,9 +488,9 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 		ctx.http_beresp = bo->beresp;
 		ctx.bo = bo;
 		ctx.now = bo->t_prev;
+		ctx.ws = bo->ws;
 	}
 	assert(ctx.now != 0);
-	ctx.ws = ws;
 	ctx.vsl = vsl;
 	ctx.method = method;
 	ctx.handling = &wrk->handling;
@@ -513,12 +514,12 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 #define VCL_MET_MAC(func, upper, bitmap)				\
 void									\
 VCL_##func##_method(struct VCL_conf *vcl, struct worker *wrk,		\
-     struct req *req, struct busyobj *bo, struct ws *ws)		\
+     struct req *req, struct busyobj *bo)				\
 {									\
 									\
 	CHECK_OBJ_NOTNULL(vcl, VCL_CONF_MAGIC);				\
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);				\
-	vcl_call_method(wrk, req, bo, ws, VCL_MET_ ## upper,		\
+	vcl_call_method(wrk, req, bo, VCL_MET_ ## upper,		\
 	    vcl->func##_func);						\
 	AN((1U << wrk->handling) & bitmap);				\
 }



From phk at FreeBSD.org  Thu May 14 09:43:27 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 11:43:27 +0200
Subject: [master] d9c1916 Add a per-method specific pointer to the vrt-context,
 and use it to slim down req/busyobj a tiny bit.
Message-ID: <E1Yspfr-0003fo-Rt@project.varnish-software.com>


commit d9c19168e544747d0f8d08499a484a883231318c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 09:42:46 2015 +0000

    Add a per-method specific pointer to the vrt-context, and use
    it to slim down req/busyobj a tiny bit.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index e51fbb4..7986cb3 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -98,7 +98,6 @@ enum {
 
 /*--------------------------------------------------------------------*/
 
-struct SHA256Context;
 struct VSC_C_lck;
 struct ban;
 struct busyobj;
@@ -518,8 +517,6 @@ struct busyobj {
 
 	struct vsl_log		vsl[1];
 
-	struct vsb		*synth_body;
-
 	uint8_t			digest[DIGEST_LEN];
 	struct vrt_privs	privs[1];
 };
@@ -592,8 +589,6 @@ struct req {
 	struct ws		ws[1];
 	struct objcore		*objcore;
 	struct objcore		*stale_oc;
-	/* Lookup stuff */
-	struct SHA256Context	*sha256ctx;
 
 	/* ESI delivery stuff */
 	ssize_t			l_crc;
@@ -619,9 +614,6 @@ struct req {
 
 	/* Temporary accounting */
 	struct acct_req		acct;
-
-	/* Synth content in vcl_synth */
-	struct vsb		*synth_body;
 };
 
 /*--------------------------------------------------------------------
@@ -1086,7 +1078,7 @@ const char *VCL_Method_Name(unsigned);
 
 #define VCL_MET_MAC(l,u,b) \
     void VCL_##l##_method(struct VCL_conf *, struct worker *, struct req *, \
-	struct busyobj *bo);
+	struct busyobj *bo, void *specific);
 #include "tbl/vcl_returns.h"
 #undef VCL_MET_MAC
 
diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c
index 34769c0..9006af8 100644
--- a/bin/varnishd/cache/cache_deliver_proc.c
+++ b/bin/varnishd/cache/cache_deliver_proc.c
@@ -115,7 +115,6 @@ VDP_DeliverObj(struct req *req)
 
 	oi = ObjIterBegin(req->wrk, req->objcore);
 	XXXAN(oi);
-	AZ(req->synth_body);
 
 	do {
 		ois = ObjIter(req->objcore, oi, &ptr, &len);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 24df6f6..b29eb25 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -274,7 +274,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(bo->vsl->wid));
 
-	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo);
+	VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL);
 
 	bo->uncacheable = bo->do_pass;
 	if (wrk->handling == VCL_RET_ABANDON)
@@ -426,7 +426,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	bo->vfc->http = bo->beresp;
 	bo->vfc->esi_req = bo->bereq;
 
-	VCL_backend_response_method(bo->vcl, wrk, NULL, bo);
+	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
 
 	if (wrk->handling == VCL_RET_ABANDON) {
 		bo->doclose = SC_RESP_CLOSE;
@@ -770,6 +770,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	ssize_t l, ll, o;
 	double now;
 	uint8_t *ptr;
+	struct vsb *synth_body;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -780,9 +781,8 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 
 	AN(bo->fetch_objcore->flags & OC_F_BUSY);
 
-	AZ(bo->synth_body);
-	bo->synth_body = VSB_new_auto();
-	AN(bo->synth_body);
+	synth_body = VSB_new_auto();
+	AN(synth_body);
 
 	// XXX: reset all beresp flags ?
 
@@ -796,13 +796,12 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	bo->fetch_objcore->exp.grace = 0;
 	bo->fetch_objcore->exp.keep = 0;
 
-	VCL_backend_error_method(bo->vcl, wrk, NULL, bo);
+	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, synth_body);
 
-	AZ(VSB_finish(bo->synth_body));
+	AZ(VSB_finish(synth_body));
 
 	if (wrk->handling == VCL_RET_RETRY) {
-		VSB_delete(bo->synth_body);
-		bo->synth_body = NULL;
+		VSB_delete(synth_body);
 
 		bo->doclose = SC_RESP_CLOSE;
 		if (bo->director_state != DIR_S_NULL)
@@ -822,22 +821,23 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	bo->vfc->http = bo->beresp;
 	bo->vfc->esi_req = bo->bereq;
 
-	if (vbf_beresp2obj(bo))
+	if (vbf_beresp2obj(bo)) {
+		VSB_delete(synth_body);
 		return (F_STP_FAIL);
+	}
 
-	ll = VSB_len(bo->synth_body);
+	ll = VSB_len(synth_body);
 	o = 0;
 	while (ll > 0) {
 		l = ll;
 		if (VFP_GetStorage(bo->vfc, &l, &ptr) != VFP_OK)
 			break;
-		memcpy(ptr, VSB_data(bo->synth_body) + o, l);
+		memcpy(ptr, VSB_data(synth_body) + o, l);
 		VBO_extend(bo, l);
 		ll -= l;
 		o += l;
 	}
-	VSB_delete(bo->synth_body);
-	bo->synth_body = NULL;
+	VSB_delete(synth_body);
 
 	HSH_Unbusy(wrk, bo->fetch_objcore);
 	VBO_setstate(bo, BOS_FINISHED);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 8a15757..03a4f77 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -187,16 +187,16 @@ HSH_DeleteObjHead(struct worker *wrk, struct objhead *oh)
 }
 
 void
-HSH_AddString(struct req *req, const char *str)
+HSH_AddString(struct req *req, void *ctx, const char *str)
 {
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	AN(req->sha256ctx);
+	AN(ctx);
 	if (str != NULL) {
-		SHA256_Update(req->sha256ctx, str, strlen(str));
+		SHA256_Update(ctx, str, strlen(str));
 		VSLb(req->vsl, SLT_Hash, "%s", str);
 	} else
-		SHA256_Update(req->sha256ctx, &str, sizeof str);
+		SHA256_Update(ctx, &str, 1);
 }
 
 /*---------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 8e48880..c4594a7 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -133,7 +133,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	    !RFC2616_Req_Gzip(req->http))
 		RFC2616_Weaken_Etag(req->resp);
 
-	VCL_deliver_method(req->vcl, wrk, req, NULL);
+	VCL_deliver_method(req->vcl, wrk, req, NULL, NULL);
 	VSLb_ts_req(req, "Process", W_TIM_real(wrk));
 
 	/* Stop the insanity before it turns "Hotel California" on us */
@@ -211,6 +211,7 @@ cnt_synth(struct worker *wrk, struct req *req)
 {
 	struct http *h;
 	double now;
+	struct vsb *synth_body;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -231,23 +232,21 @@ cnt_synth(struct worker *wrk, struct req *req)
 	http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid));
 	http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
 
-	AZ(req->synth_body);
-	req->synth_body = VSB_new_auto();
-	AN(req->synth_body);
+	synth_body = VSB_new_auto();
+	AN(synth_body);
 
-	VCL_synth_method(req->vcl, wrk, req, NULL);
+	VCL_synth_method(req->vcl, wrk, req, NULL, synth_body);
 
 	http_Unset(h, H_Content_Length);
 
-	AZ(VSB_finish(req->synth_body));
+	AZ(VSB_finish(synth_body));
 
 	/* Discard any lingering request body before delivery */
 	(void)VRB_Ignore(req);
 
 	if (wrk->handling == VCL_RET_RESTART) {
 		HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
-		VSB_delete(req->synth_body);
-		req->synth_body = NULL;
+		VSB_delete(synth_body);
 		req->req_step = R_STP_RESTART;
 		return (REQ_FSM_MORE);
 	}
@@ -262,21 +261,20 @@ cnt_synth(struct worker *wrk, struct req *req)
 		ssize_t sz, szl;
 		uint8_t *ptr;
 
-		szl = VSB_len(req->synth_body);
+		szl = VSB_len(synth_body);
 		assert(szl >= 0);
 		if (szl > 0) {
 			sz = szl;
 			AN(ObjGetSpace(wrk, req->objcore, &sz, &ptr));
 			assert(sz >= szl);
-			memcpy(ptr, VSB_data(req->synth_body), szl);
+			memcpy(ptr, VSB_data(synth_body), szl);
 			ObjExtend(wrk, req->objcore, szl);
 		}
-		VSB_delete(req->synth_body);
-		req->synth_body = NULL;
 
 		cnt_vdp(req, NULL);
 		(void)HSH_DerefObjCore(wrk, &req->objcore);
 	}
+	VSB_delete(synth_body);
 
 	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
 
@@ -388,7 +386,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 
 	VSLb(req->vsl, SLT_Hit, "%u", ObjGetXID(wrk, req->objcore));
 
-	VCL_hit_method(req->vcl, wrk, req, NULL);
+	VCL_hit_method(req->vcl, wrk, req, NULL, NULL);
 
 	switch (wrk->handling) {
 	case VCL_RET_DELIVER:
@@ -463,7 +461,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
-	VCL_miss_method(req->vcl, wrk, req, NULL);
+	VCL_miss_method(req->vcl, wrk, req, NULL, NULL);
 	switch (wrk->handling) {
 	case VCL_RET_FETCH:
 		wrk->stats->cache_miss++;
@@ -504,7 +502,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(req->objcore);
 
-	VCL_pass_method(req->vcl, wrk, req, NULL);
+	VCL_pass_method(req->vcl, wrk, req, NULL, NULL);
 	switch (wrk->handling) {
 	case VCL_RET_SYNTH:
 		req->req_step = R_STP_SYNTH;
@@ -550,7 +548,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(req->vsl->wid));
 	http_SetHeader(bo->bereq, "Connection: close");
 
-	VCL_pipe_method(req->vcl, wrk, req, bo);
+	VCL_pipe_method(req->vcl, wrk, req, bo, NULL);
 
 	if (wrk->handling == VCL_RET_SYNTH)
 		INCOMPL();
@@ -651,7 +649,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 
 	http_CollectHdr(req->http, H_Cache_Control);
 
-	VCL_recv_method(req->vcl, wrk, req, NULL);
+	VCL_recv_method(req->vcl, wrk, req, NULL, NULL);
 
 	/* Attempts to cache req.body may fail */
 	if (req->req_body_status == REQ_BODY_FAIL) {
@@ -672,12 +670,10 @@ cnt_recv(struct worker *wrk, struct req *req)
 		}
 	}
 
-	req->sha256ctx = &sha256ctx;	/* so HSH_AddString() can find it */
-	SHA256_Init(req->sha256ctx);
-	VCL_hash_method(req->vcl, wrk, req, NULL);
+	SHA256_Init(&sha256ctx);
+	VCL_hash_method(req->vcl, wrk, req, NULL, &sha256ctx);
 	assert(wrk->handling == VCL_RET_LOOKUP);
-	SHA256_Final(req->digest, req->sha256ctx);
-	req->sha256ctx = NULL;
+	SHA256_Final(req->digest, &sha256ctx);
 
 	if (!strcmp(req->http->hd[HTTP_HDR_METHOD].b, "HEAD"))
 		req->wantbody = 0;
@@ -743,7 +739,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 
 	AZ(HSH_DerefObjCore(wrk, &boc));
 
-	VCL_purge_method(req->vcl, wrk, req, NULL);
+	VCL_purge_method(req->vcl, wrk, req, NULL, NULL);
 	switch (wrk->handling) {
 	case VCL_RET_RESTART:
 		req->req_step = R_STP_RESTART;
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index ac051d1..a453cf3 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -461,7 +461,7 @@ ccf_config_show(struct cli *cli, const char * const *av, void *priv)
 
 static void
 vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
-    unsigned method, vcl_func_f *func)
+    void *specific, unsigned method, vcl_func_f *func)
 {
 	char *aws;
 	struct vsl_log *vsl = NULL;
@@ -492,6 +492,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 	}
 	assert(ctx.now != 0);
 	ctx.vsl = vsl;
+	ctx.specific = specific;
 	ctx.method = method;
 	ctx.handling = &wrk->handling;
 	aws = WS_Snapshot(wrk->aws);
@@ -514,13 +515,13 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 #define VCL_MET_MAC(func, upper, bitmap)				\
 void									\
 VCL_##func##_method(struct VCL_conf *vcl, struct worker *wrk,		\
-     struct req *req, struct busyobj *bo)				\
+     struct req *req, struct busyobj *bo, void *specific)		\
 {									\
 									\
 	CHECK_OBJ_NOTNULL(vcl, VCL_CONF_MAGIC);				\
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);				\
-	vcl_call_method(wrk, req, bo, VCL_MET_ ## upper,		\
-	    vcl->func##_func);						\
+	vcl_call_method(wrk, req, bo, specific,				\
+	    VCL_MET_ ## upper, vcl->func##_func);			\
 	AN((1U << wrk->handling) & bitmap);				\
 }
 
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 63d2502..5a12c63 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -264,20 +264,21 @@ VRT_hashdata(VRT_CTX, const char *str, ...)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	HSH_AddString(ctx->req, str);
+	AN(ctx->specific);
+	HSH_AddString(ctx->req, ctx->specific, str);
 	va_start(ap, str);
 	while (1) {
 		p = va_arg(ap, const char *);
 		if (p == vrt_magic_string_end)
 			break;
-		HSH_AddString(ctx->req, p);
+		HSH_AddString(ctx->req, ctx->specific, p);
 	}
 	va_end(ap);
 	/*
 	 * Add a 'field-separator' to make it more difficult to
 	 * manipulate the hash.
 	 */
-	HSH_AddString(ctx->req, NULL);
+	HSH_AddString(ctx->req, ctx->specific, NULL);
 }
 
 /*--------------------------------------------------------------------*/
@@ -386,16 +387,7 @@ VRT_synth_page(VRT_CTX, const char *str, ...)
 	const char *p;
 	struct vsb *vsb;
 
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	if (ctx->method == VCL_MET_BACKEND_ERROR) {
-		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-		vsb = ctx->bo->synth_body;
-	} else {
-		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-		vsb = ctx->req->synth_body;
-	}
-	AN(vsb);
-
+	CAST_OBJ_NOTNULL(vsb, ctx->specific, VSB_MAGIC);
 	va_start(ap, str);
 	p = str;
 	while (p != vrt_magic_string_end) {
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 6f546ff..47a68cb 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -67,7 +67,7 @@ enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **,
     int wait_for_busy, int always_insert);
 void HSH_Ref(struct objcore *o);
 void HSH_Init(const struct hash_slinger *slinger);
-void HSH_AddString(struct req *, const char *str);
+void HSH_AddString(struct req *, void *ctx, const char *str);
 void HSH_Insert(struct worker *, const void *hash, struct objcore *);
 void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace,
     double keep);
diff --git a/include/vrt.h b/include/vrt.h
index 5e4882e..97c4e92 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -107,6 +107,13 @@ struct vrt_ctx {
 	struct http			*http_beresp;
 
 	double				now;
+
+	/*
+	 * method specific argument:
+	 *    hash:		struct SHA256ctx
+	 *    synth+error:	struct vsb *
+	 */
+	void				*specific;
 };
 
 #define VRT_CTX		const struct vrt_ctx *ctx



From phk at FreeBSD.org  Thu May 14 15:11:29 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 17:11:29 +0200
Subject: [master] b588787 Fix comment.
Message-ID: <E1YsunJ-0002Cd-TE@project.varnish-software.com>


commit b58878745aafe6a5d4747867b79fd7ae585618d5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 15:11:10 2015 +0000

    Fix comment.
    
    Submitted by: Dridi

diff --git a/include/vrt.h b/include/vrt.h
index 97c4e92..785dd8c 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -110,7 +110,7 @@ struct vrt_ctx {
 
 	/*
 	 * method specific argument:
-	 *    hash:		struct SHA256ctx
+	 *    hash:		struct SHA256Context
 	 *    synth+error:	struct vsb *
 	 */
 	void				*specific;



From phk at FreeBSD.org  Thu May 14 16:12:23 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 18:12:23 +0200
Subject: [master] 3cc846e Introduce "transports" which is how responses to
 requests gets delivered.
Message-ID: <E1YsvkF-0002Lb-0K@project.varnish-software.com>


commit 3cc846eea3c42c11f73bd4890b47723241a1b443
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 16:09:50 2015 +0000

    Introduce "transports" which is how responses to requests gets
    delivered.
    
    Transports are a superset of protocols because internal requests,
    notably ESI:includes, doesn't use a HTTP protocol to deliver their
    response, but have have ?dipal ways of doing things.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7986cb3..2f0d228 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -111,6 +111,7 @@ struct object;
 struct objhead;
 struct pool;
 struct poolparam;
+struct transport;
 struct req;
 struct sess;
 struct suckaddr;
@@ -541,6 +542,10 @@ struct req {
 	struct sess		*sp;
 	struct worker		*wrk;
 	struct pool_task	task;
+
+	const struct transport	*transport;
+	void			*transport_priv;
+
 	enum req_step		req_step;
 	VTAILQ_ENTRY(req)	w_list;
 
@@ -672,6 +677,21 @@ struct sess {
 
 };
 
+/*--------------------------------------------------------------------
+ * A transport is how we talk HTTP for a given request.
+ * This is different from a protocol because ESI child requests have
+ * their own "protocol" to talk to the parent ESI request, which may
+ * or may not, be talking a "real" HTTP protocol itself.
+ */
+
+typedef void vtr_deliver_f (struct req *);
+
+struct transport {
+	unsigned		magic;
+#define TRANSPORT_MAGIC		0xf157f32f
+	vtr_deliver_f		*deliver;
+};
+
 /* Prototypes etc ----------------------------------------------------*/
 
 /* Cross file typedefs */
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c4594a7..8f0f1a4 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -80,7 +80,8 @@ cnt_vdp(struct req *req, struct busyobj *bo)
 			VRG_dorange(req, r);
 	}
 
-	V1D_Deliver(req);
+	CHECK_OBJ_NOTNULL(req->transport, TRANSPORT_MAGIC);
+	req->transport->deliver(req);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index 984a3f1..221173d 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -60,7 +60,7 @@ v1d_bytes(struct req *req, enum vdp_action act, void **priv,
 /*--------------------------------------------------------------------
  */
 
-void
+void __match_proto__(vtr_deliver_f)
 V1D_Deliver(struct req *req)
 {
 	enum objiter_status ois;
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index 9d08d29..8954061 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -42,6 +42,11 @@
 
 #include "vtcp.h"
 
+static const struct transport http1_transport = {
+	.magic = TRANSPORT_MAGIC,
+	.deliver = V1D_Deliver,
+};
+
 /*----------------------------------------------------------------------
  */
 
@@ -255,10 +260,12 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			sp->sess_step = S_STP_H1PROC;
 			break;
 		case S_STP_H1PROC:
+			req->transport = &http1_transport;
 			if (CNT_Request(wrk, req) == REQ_FSM_DISEMBARK) {
 				sp->sess_step = S_STP_H1BUSY;
 				return;
 			}
+			req->transport = NULL;
 			sp->sess_step = S_STP_H1CLEANUP;
 			break;
 		case S_STP_H1CLEANUP:



From phk at FreeBSD.org  Thu May 14 16:17:46 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 18:17:46 +0200
Subject: [master] ca50ccd Start using the "transport" mechanism for esi
 includes
Message-ID: <E1YsvpS-0003G0-UN@project.varnish-software.com>


commit ca50ccd55d9cc6fa1b25e10ae3383b336e7bc81d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 16:17:25 2015 +0000

    Start using the "transport" mechanism for esi includes

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 7f072ea..087af87 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -70,6 +70,7 @@ ved_include(struct req *preq, const char *src, const char *host)
 	struct worker *wrk;
 	struct req *req;
 	enum req_fsm_nxt s;
+	struct transport xp;
 
 	wrk = preq->wrk;
 
@@ -134,7 +135,9 @@ ved_include(struct req *preq, const char *src, const char *host)
 	req->crc = preq->crc;
 	req->l_crc = preq->l_crc;
 
-	VDP_push(req, ved_vdp_bytes, preq, 0);
+	INIT_OBJ(&xp, TRANSPORT_MAGIC);
+	req->transport = &xp;
+	req->transport_priv = preq;
 
 	THR_SetRequest(req);
 
@@ -657,6 +660,7 @@ int
 VED_Setup(struct req *req, struct busyobj *bo)
 {
 	int i;
+	struct req *preq;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
@@ -677,6 +681,10 @@ VED_Setup(struct req *req, struct busyobj *bo)
 	if (req->esi_level == 0)
 		return (0);
 
+	CAST_OBJ_NOTNULL(preq, req->transport_priv, REQ_MAGIC);
+
+	VDP_push(req, ved_vdp_bytes, preq, 1);
+
 	req->res_mode |= RES_ESI_CHILD;
 	i = ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED);
 	if (req->gzip_resp && i && !(req->res_mode & RES_ESI)) {



From phk at FreeBSD.org  Thu May 14 17:16:02 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 19:16:02 +0200
Subject: [master] 8b1f899 Make ESI delivery less magic.
Message-ID: <E1Yswjq-0001Jt-Bp@project.varnish-software.com>


commit 8b1f8991a3d54c7f1cd39f7995bfb1d00f80753f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 17:15:24 2015 +0000

    Make ESI delivery less magic.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2f0d228..17f59dd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -684,7 +684,7 @@ struct sess {
  * or may not, be talking a "real" HTTP protocol itself.
  */
 
-typedef void vtr_deliver_f (struct req *);
+typedef void vtr_deliver_f (struct req *, struct busyobj *);
 
 struct transport {
 	unsigned		magic;
@@ -746,7 +746,7 @@ extern const int HTTP1_Req[3];
 extern const int HTTP1_Resp[3];
 
 /* cache_http1_deliver.c */
-void V1D_Deliver(struct req *);
+vtr_deliver_f V1D_Deliver;
 
 /* cache_http1_pipe.c */
 void V1P_Init(void);
@@ -1113,8 +1113,6 @@ char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
 void VRTPRIV_init(struct vrt_privs *privs);
 void VRTPRIV_dynamic_kill(struct vrt_privs *privs, uintptr_t id);
 
-int VED_Setup(struct req *req, struct busyobj *bo);
-
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);
 
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 087af87..356d0ea 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -41,6 +41,8 @@
 #include "vend.h"
 #include "vgz.h"
 
+static vtr_deliver_f VED_Deliver;
+
 /*--------------------------------------------------------------------*/
 
 static int __match_proto__(vdp_bytes)
@@ -108,10 +110,18 @@ ved_include(struct req *preq, const char *src, const char *host)
 	http_ForceField(req->http0, HTTP_HDR_METHOD, "GET");
 	http_ForceField(req->http0, HTTP_HDR_PROTO, "HTTP/1.1");
 
-	/* Don't allow Conditions, we can't use a 304 */
+	/* Don't allow conditionalss, we can't use a 304 */
 	http_Unset(req->http0, H_If_Modified_Since);
 	http_Unset(req->http0, H_If_None_Match);
 
+	/* Don't allow Range */
+	http_Unset(req->http0, H_Range);
+
+	/* Set Accept-Encoding according to what we want */
+	http_Unset(req->http0, H_Accept_Encoding);
+	if (preq->gzip_resp)
+		http_ForceHeader(req->http0, H_Accept_Encoding, "gzip");
+
 	/* Client content already taken care of */
 	http_Unset(req->http0, H_Content_Length);
 
@@ -136,6 +146,7 @@ ved_include(struct req *preq, const char *src, const char *host)
 	req->l_crc = preq->l_crc;
 
 	INIT_OBJ(&xp, TRANSPORT_MAGIC);
+	xp.deliver = VED_Deliver;
 	req->transport = &xp;
 	req->transport_priv = preq;
 
@@ -656,50 +667,33 @@ ved_stripgzip(struct req *req)
 	req->l_crc += ilen;
 }
 
-int
-VED_Setup(struct req *req, struct busyobj *bo)
+static void __match_proto__(vtr_deliver_f)
+VED_Deliver(struct req *req, struct busyobj *bo)
 {
 	int i;
 	struct req *preq;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
-	/*
-	 * Determine ESI status first.  Not dependent on wantbody, because
-	 * we want ESI to supress C-L in HEAD too.
-	 */
-	if (!req->disable_esi &&
-	    ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, NULL) != NULL) {
-		req->res_mode |= RES_ESI;
-		RFC2616_Weaken_Etag(req->resp);
-		req->resp_len = -1;
-		VDP_push(req, VDP_ESI, NULL, 0);
-	}
-
-	/* ESI-childen need special treatment */
-	if (req->esi_level == 0)
-		return (0);
-
 	CAST_OBJ_NOTNULL(preq, req->transport_priv, REQ_MAGIC);
 
-	VDP_push(req, ved_vdp_bytes, preq, 1);
-
 	req->res_mode |= RES_ESI_CHILD;
 	i = ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED);
 	if (req->gzip_resp && i && !(req->res_mode & RES_ESI)) {
+		VDP_push(req, ved_vdp_bytes, preq, 1);
+
 		if (bo != NULL)
 			VBO_waitstate(bo, BOS_FINISHED);
 		ved_stripgzip(req);
 		(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
 	} else {
 		if (req->gzip_resp && !i)
-			VDP_push(req, ved_pretend_gzip, NULL, 0);
-		else if (!req->gzip_resp && i)
-			VDP_push(req, VDP_gunzip, NULL, 0);
+			VDP_push(req, ved_pretend_gzip, NULL, 1);
 
+		VDP_push(req, ved_vdp_bytes, preq, 1);
 		(void)VDP_DeliverObj(req);
 	}
 	VDP_close(req);
-	return (1);
 }
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 8f0f1a4..1f0e306 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -58,8 +58,18 @@ cnt_vdp(struct req *req, struct busyobj *bo)
 	else
 		req->resp_len = ObjGetLen(req->wrk, req->objcore);
 
-	if (VED_Setup(req, bo))
-		return;
+	/*
+	 * Determine ESI status first.  Not dependent on wantbody, because
+	 * we want ESI to supress C-L in HEAD too.
+	 */
+	if (!req->disable_esi &&
+	    ObjGetattr(req->wrk, req->objcore, OA_ESIDATA, NULL) != NULL) {
+		req->res_mode |= RES_ESI;
+		RFC2616_Weaken_Etag(req->resp);
+		req->resp_len = -1;
+		VDP_push(req, VDP_ESI, NULL, 0);
+	}
+
 
 	if (cache_param->http_gzip_support &&
 	    ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
@@ -81,7 +91,7 @@ cnt_vdp(struct req *req, struct busyobj *bo)
 	}
 
 	CHECK_OBJ_NOTNULL(req->transport, TRANSPORT_MAGIC);
-	req->transport->deliver(req);
+	req->transport->deliver(req, bo);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index 221173d..710dee2 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -61,11 +61,12 @@ v1d_bytes(struct req *req, enum vdp_action act, void **priv,
  */
 
 void __match_proto__(vtr_deliver_f)
-V1D_Deliver(struct req *req)
+V1D_Deliver(struct req *req, struct busyobj *bo)
 {
 	enum objiter_status ois;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
 	if ((req->objcore->flags & OC_F_PRIVATE) &&



From phk at FreeBSD.org  Thu May 14 17:16:02 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 19:16:02 +0200
Subject: [master] 6764fbb Add a new test-case.
Message-ID: <E1Yswjq-0001K3-D2@project.varnish-software.com>


commit 6764fbb560ae3339c7b50679fc48b121a78b8d04
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 17:15:40 2015 +0000

    Add a new test-case.

diff --git a/bin/varnishtest/tests/e00029.vtc b/bin/varnishtest/tests/e00029.vtc
new file mode 100644
index 0000000..5be8aa3
--- /dev/null
+++ b/bin/varnishtest/tests/e00029.vtc
@@ -0,0 +1,26 @@
+varnishtest "fun esi includes and ranges"
+
+server s1 {
+	rxreq
+	txresp -body {<html><esi:include src="/bar"/></html>}
+	rxreq 
+	expect req.url == /bar
+	txresp -body "ABCD"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.url == "/bar") {
+			set req.http.Range = "bytes=1-2";
+		}
+	}
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.body == "<html>BC</html>"
+} -run
diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc
index 1418212..712d51b 100644
--- a/bin/varnishtest/tests/g00002.vtc
+++ b/bin/varnishtest/tests/g00002.vtc
@@ -42,6 +42,7 @@ client c1 {
 	rxresp
 	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 4100
+	delay .1
 
 	# See varnish can gunzip it, inside ESI
 	txreq -url /bar -hdr "Accept-Encoding: null"



From phk at FreeBSD.org  Thu May 14 17:30:34 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 19:30:34 +0200
Subject: [master] 01bd798 Shuffle order of functions
Message-ID: <E1Yswxu-0002xW-KT@project.varnish-software.com>


commit 01bd798586c30e2d7d74932d686e88532d5837c4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 17:21:22 2015 +0000

    Shuffle order of functions

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 356d0ea..637ee4c 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -220,67 +220,6 @@ ved_decode_len(uint8_t **pp)
 }
 
 /*---------------------------------------------------------------------
- * If a gzip'ed ESI object includes a ungzip'ed object, we need to make
- * it looked like a gzip'ed data stream.  The official way to do so would
- * be to fire up libvgz and gzip it, but we don't, we fake it.
- *
- * First, we cannot know if it is ungzip'ed on purpose, the admin may
- * know something we don't.
- *
- * What do you mean "BS ?"
- *
- * All right then...
- *
- * The matter of the fact is that we simply will not fire up a gzip in
- * the output path because it costs too much memory and CPU, so we simply
- * wrap the data in very convenient "gzip copy-blocks" and send it down
- * the stream with a bit more overhead.
- */
-
-static int __match_proto__(vdp_bytes)
-ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv,
-    const void *pv, ssize_t l)
-{
-	uint8_t buf1[5], buf2[5];
-	const uint8_t *p;
-	uint16_t lx;
-
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	(void)priv;
-	if (act == VDP_INIT || act == VDP_FINI)
-		return (0);
-	p = pv;
-
-	lx = 65535;
-	buf1[0] = 0;
-	vle16enc(buf1 + 1, lx);
-	vle16enc(buf1 + 3, ~lx);
-
-	while (l > 0) {
-		if (l >= 65535) {
-			lx = 65535;
-			if (VDP_bytes(req, VDP_NULL, buf1, sizeof buf1))
-				return (-1);
-		} else {
-			lx = (uint16_t)l;
-			buf2[0] = 0;
-			vle16enc(buf2 + 1, lx);
-			vle16enc(buf2 + 3, ~lx);
-			if (VDP_bytes(req, VDP_NULL, buf2, sizeof buf2))
-				return (-1);
-		}
-		if (VDP_bytes(req, VDP_NULL, p, lx))
-			return (-1);
-		req->crc = crc32(req->crc, p, lx);
-		req->l_crc += lx;
-		l -= lx;
-		p += lx;
-	}
-	/* buf2 is local, have to flush */
-	return (VDP_bytes(req, VDP_FLUSH, NULL, 0));
-}
-
-/*---------------------------------------------------------------------
  */
 
 static const uint8_t gzip_hdr[] = {
@@ -474,6 +413,67 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 }
 
 /*---------------------------------------------------------------------
+ * If a gzip'ed ESI object includes a ungzip'ed object, we need to make
+ * it looked like a gzip'ed data stream.  The official way to do so would
+ * be to fire up libvgz and gzip it, but we don't, we fake it.
+ *
+ * First, we cannot know if it is ungzip'ed on purpose, the admin may
+ * know something we don't.
+ *
+ * What do you mean "BS ?"
+ *
+ * All right then...
+ *
+ * The matter of the fact is that we simply will not fire up a gzip in
+ * the output path because it costs too much memory and CPU, so we simply
+ * wrap the data in very convenient "gzip copy-blocks" and send it down
+ * the stream with a bit more overhead.
+ */
+
+static int __match_proto__(vdp_bytes)
+ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv,
+    const void *pv, ssize_t l)
+{
+	uint8_t buf1[5], buf2[5];
+	const uint8_t *p;
+	uint16_t lx;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	(void)priv;
+	if (act == VDP_INIT || act == VDP_FINI)
+		return (0);
+	p = pv;
+
+	lx = 65535;
+	buf1[0] = 0;
+	vle16enc(buf1 + 1, lx);
+	vle16enc(buf1 + 3, ~lx);
+
+	while (l > 0) {
+		if (l >= 65535) {
+			lx = 65535;
+			if (VDP_bytes(req, VDP_NULL, buf1, sizeof buf1))
+				return (-1);
+		} else {
+			lx = (uint16_t)l;
+			buf2[0] = 0;
+			vle16enc(buf2 + 1, lx);
+			vle16enc(buf2 + 3, ~lx);
+			if (VDP_bytes(req, VDP_NULL, buf2, sizeof buf2))
+				return (-1);
+		}
+		if (VDP_bytes(req, VDP_NULL, p, lx))
+			return (-1);
+		req->crc = crc32(req->crc, p, lx);
+		req->l_crc += lx;
+		l -= lx;
+		p += lx;
+	}
+	/* buf2 is local, have to flush */
+	return (VDP_bytes(req, VDP_FLUSH, NULL, 0));
+}
+
+/*---------------------------------------------------------------------
  * Include an object in a gzip'ed ESI object delivery
  */
 



From phk at FreeBSD.org  Thu May 14 17:30:34 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 19:30:34 +0200
Subject: [master] 68dc19d More reordering
Message-ID: <E1Yswxu-0002xd-Lj@project.varnish-software.com>


commit 68dc19d1d8a030c7531ae80bb8dfe760f11a738d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 17:30:23 2015 +0000

    More reordering

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 637ee4c..9670d19 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -43,26 +43,22 @@
 
 static vtr_deliver_f VED_Deliver;
 
-/*--------------------------------------------------------------------*/
-
-static int __match_proto__(vdp_bytes)
-ved_vdp_bytes(struct req *req, enum vdp_action act, void **priv,
-    const void *ptr, ssize_t len)
-{
-	struct req *preq;
-
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	if (act == VDP_INIT)
-		return (0);
-	if (act == VDP_FINI) {
-		*priv = NULL;
-		return (0);
-	}
-	CAST_OBJ_NOTNULL(preq, *priv, REQ_MAGIC);
-	req->acct.resp_bodybytes += len;
-	return (VDP_bytes(preq, act, ptr, len));
-}
+static const uint8_t gzip_hdr[] = {
+	0x1f, 0x8b, 0x08,
+	0x00, 0x00, 0x00, 0x00,
+	0x00,
+	0x02, 0x03
+};
 
+struct ecx {
+	unsigned	magic;
+#define ECX_MAGIC	0x0b0f9163
+	uint8_t		*p;
+	uint8_t		*e;
+	int		state;
+	ssize_t		l;
+	int		isgzip;
+};
 
 /*--------------------------------------------------------------------*/
 
@@ -186,7 +182,6 @@ ved_include(struct req *preq, const char *src, const char *host)
 
 /*--------------------------------------------------------------------*/
 
-
 //#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
 #define Debug(fmt, ...) /**/
 
@@ -222,23 +217,6 @@ ved_decode_len(uint8_t **pp)
 /*---------------------------------------------------------------------
  */
 
-static const uint8_t gzip_hdr[] = {
-	0x1f, 0x8b, 0x08,
-	0x00, 0x00, 0x00, 0x00,
-	0x00,
-	0x02, 0x03
-};
-
-struct ecx {
-	unsigned	magic;
-#define ECX_MAGIC	0x0b0f9163
-	uint8_t		*p;
-	uint8_t		*e;
-	int		state;
-	ssize_t		l;
-	int		isgzip;
-};
-
 int __match_proto__(vdp_bytes)
 VDP_ESI(struct req *req, enum vdp_action act, void **priv,
     const void *ptr, ssize_t len)
@@ -667,6 +645,28 @@ ved_stripgzip(struct req *req)
 	req->l_crc += ilen;
 }
 
+/*--------------------------------------------------------------------*/
+
+static int __match_proto__(vdp_bytes)
+ved_vdp_bytes(struct req *req, enum vdp_action act, void **priv,
+    const void *ptr, ssize_t len)
+{
+	struct req *preq;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	if (act == VDP_INIT)
+		return (0);
+	if (act == VDP_FINI) {
+		*priv = NULL;
+		return (0);
+	}
+	CAST_OBJ_NOTNULL(preq, *priv, REQ_MAGIC);
+	req->acct.resp_bodybytes += len;
+	return (VDP_bytes(preq, act, ptr, len));
+}
+
+/*--------------------------------------------------------------------*/
+
 static void __match_proto__(vtr_deliver_f)
 VED_Deliver(struct req *req, struct busyobj *bo)
 {



From phk at FreeBSD.org  Thu May 14 20:12:50 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 22:12:50 +0200
Subject: [master] e3712a4 Slight improvement to our symbol lookup hack.
Message-ID: <E1YszUw-0004Ir-HY@project.varnish-software.com>


commit e3712a45ed2f6d89fb5b814a8681a1810596c8a1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 17:54:46 2015 +0000

    Slight improvement to our symbol lookup hack.

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index c5413ad..533ec80 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -233,14 +233,16 @@ Symbol_Lookup(struct vsb *vsb, void *ptr)
 	pp = (uintptr_t)ptr;
 	s0 = NULL;
 	VTAILQ_FOREACH(s, &symbols, list) {
-		if (s->a > pp || s->a + s->l < pp)
+		if (s->a > pp || s->a + s->l <= pp)
 			continue;
 		if (s0 == NULL || s->l < s0->l)
 			s0 = s;
 	}
 	if (s0 == NULL)
 		return (-1);
-	VSB_printf(vsb, "%p: %s+0x%jx", ptr, s0->n, (uintmax_t)pp - s0->a);
+	VSB_printf(vsb, "%p: %s", ptr, s0->n);
+	if ((uintmax_t)pp != s0->a)
+		VSB_printf(vsb, "+0x%jx", (uintmax_t)pp - s0->a);
 	return (0);
 }
 



From phk at FreeBSD.org  Thu May 14 20:12:50 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 22:12:50 +0200
Subject: [master] 3341de5 Straighten out ESI delivery some more.
Message-ID: <E1YszUw-0004Ix-Ie@project.varnish-software.com>


commit 3341de57cf7b3704e6d7a6d80925ad01622010cc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 20:12:18 2015 +0000

    Straighten out ESI delivery some more.

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 9670d19..02c992f 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -63,13 +63,16 @@ struct ecx {
 /*--------------------------------------------------------------------*/
 
 static void
-ved_include(struct req *preq, const char *src, const char *host)
+ved_include(struct req *preq, const char *src, const char *host,
+    struct ecx *ecx)
 {
 	struct worker *wrk;
 	struct req *req;
 	enum req_fsm_nxt s;
 	struct transport xp;
 
+	CHECK_OBJ_NOTNULL(preq, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(ecx, ECX_MAGIC);
 	wrk = preq->wrk;
 
 	if (preq->esi_level >= cache_param->max_esi_depth)
@@ -115,7 +118,7 @@ ved_include(struct req *preq, const char *src, const char *host)
 
 	/* Set Accept-Encoding according to what we want */
 	http_Unset(req->http0, H_Accept_Encoding);
-	if (preq->gzip_resp)
+	if (ecx->isgzip)
 		http_ForceHeader(req->http0, H_Accept_Encoding, "gzip");
 
 	/* Client content already taken care of */
@@ -137,9 +140,7 @@ ved_include(struct req *preq, const char *src, const char *host)
 	req->t_req = preq->t_req;
 	assert(isnan(req->t_first));
 	assert(isnan(req->t_prev));
-	req->gzip_resp = preq->gzip_resp;
-	req->crc = preq->crc;
-	req->l_crc = preq->l_crc;
+	req->gzip_resp = ecx->isgzip;
 
 	INIT_OBJ(&xp, TRANSPORT_MAGIC);
 	xp.deliver = VED_Deliver;
@@ -171,8 +172,11 @@ ved_include(struct req *preq, const char *src, const char *host)
 	preq->vcl = req->vcl;
 	req->vcl = NULL;
 
-	preq->crc = req->crc;
-	preq->l_crc = req->l_crc;
+	if (ecx->isgzip && req->l_crc) {
+		preq->crc = crc32_combine(
+		    preq->crc, req->crc, req->l_crc);
+		preq->l_crc += req->l_crc;
+	}
 
 	req->wrk = NULL;
 
@@ -323,7 +327,7 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 				}
 				Debug("INCL [%s][%s] BEGIN\n", q, ecx->p);
 				ved_include(req,
-				    (const char*)q, (const char*)ecx->p);
+				    (const char*)q, (const char*)ecx->p, ecx);
 				Debug("INCL [%s][%s] END\n", q, ecx->p);
 				ecx->p = r + 1;
 				break;



From phk at FreeBSD.org  Thu May 14 20:12:50 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 22:12:50 +0200
Subject: [master] fc53691 Make this test-case even more strange.
Message-ID: <E1YszUw-0004J6-Jc@project.varnish-software.com>


commit fc53691edb65e9f5c5eab82aae6ab4a1c213afc2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 20:12:34 2015 +0000

    Make this test-case even more strange.

diff --git a/bin/varnishtest/tests/e00029.vtc b/bin/varnishtest/tests/e00029.vtc
index 5be8aa3..ea3dd70 100644
--- a/bin/varnishtest/tests/e00029.vtc
+++ b/bin/varnishtest/tests/e00029.vtc
@@ -2,16 +2,27 @@ varnishtest "fun esi includes and ranges"
 
 server s1 {
 	rxreq
+	expect req.url == "/1"
 	txresp -body {<html><esi:include src="/bar"/></html>}
-	rxreq 
+
+	rxreq
 	expect req.url == /bar
+	txresp -body {<html><esi:include src="/foo"/></html>}
+
+	rxreq
+	expect req.url == /foo
 	txresp -body "ABCD"
+
+	rxreq
+	expect req.url == "/2"
+	txresp -gzipbody {<html><esi:include src="/bar"/></html>}
+
 } -start
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.url == "/bar") {
-			set req.http.Range = "bytes=1-2";
+			set req.http.Range = "bytes=7-8";
 		}
 	}
 	sub vcl_backend_response {
@@ -20,7 +31,12 @@ varnish v1 -vcl+backend {
 } -start
 
 client c1 {
-	txreq
+	txreq -url /1 -hdr "Accept-encoding: gzip"
+	rxresp
+	expect resp.body == "<html>BC</html>"
+	delay .1
+	txreq -url /2 -hdr "Accept-encoding: gzip"
 	rxresp
+	gunzip
 	expect resp.body == "<html>BC</html>"
 } -run



From phk at FreeBSD.org  Thu May 14 20:43:48 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 22:43:48 +0200
Subject: [master] ede0ff5 Gross hack, but could be handy another time.
Message-ID: <E1Yszyu-0007ha-UT@project.varnish-software.com>


commit ede0ff58327553d470fc5f2ce891c8a72e9f4d42
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 20:42:48 2015 +0000

    Gross hack, but could be handy another time.

diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c
index 9006af8..4fd666b 100644
--- a/bin/varnishd/cache/cache_deliver_proc.c
+++ b/bin/varnishd/cache/cache_deliver_proc.c
@@ -63,6 +63,16 @@ VDP_push(struct req *req, vdp_bytes *func, void *priv, int bottom)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	AN(func);
 
+#if 0
+	// Gross hack for debugging
+	struct vsb *vsb;
+	vsb = VSB_new_auto();
+	AN(vsb);
+	Symbol_Lookup(vsb, func);
+	AZ(VSB_finish(vsb));
+	VSLb(req->vsl, SLT_Debug, "VDP %s %p %d", VSB_data(vsb), priv, bottom);
+	VSB_delete(vsb);
+#endif
 	vdp = WS_Alloc(req->ws, sizeof *vdp);
 	AN(vdp);
 	INIT_OBJ(vdp, VDP_ENTRY_MAGIC);



From phk at FreeBSD.org  Thu May 14 20:43:48 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 22:43:48 +0200
Subject: [master] 4ccf3b5 Remove the ESI/gzip CRC fields from struct req,
 and pass them as priv's inside ESI's delivery code.
Message-ID: <E1Yszyu-0007hg-Va@project.varnish-software.com>


commit 4ccf3b5ba7f834beccd708627970aa9fab0d33cb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 20:43:16 2015 +0000

    Remove the ESI/gzip CRC fields from struct req, and pass them
    as priv's inside ESI's delivery code.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 17f59dd..f9e5326 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -595,10 +595,6 @@ struct req {
 	struct objcore		*objcore;
 	struct objcore		*stale_oc;
 
-	/* ESI delivery stuff */
-	ssize_t			l_crc;
-	uint32_t		crc;
-
 	/* Delivery mode */
 	unsigned		res_mode;
 #define RES_LEN			(1<<1)
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 02c992f..5bb360a 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -58,6 +58,10 @@ struct ecx {
 	int		state;
 	ssize_t		l;
 	int		isgzip;
+
+	struct req	*preq;
+	ssize_t		l_crc;
+	uint32_t	crc;
 };
 
 /*--------------------------------------------------------------------*/
@@ -145,7 +149,7 @@ ved_include(struct req *preq, const char *src, const char *host,
 	INIT_OBJ(&xp, TRANSPORT_MAGIC);
 	xp.deliver = VED_Deliver;
 	req->transport = &xp;
-	req->transport_priv = preq;
+	req->transport_priv = ecx;
 
 	THR_SetRequest(req);
 
@@ -172,12 +176,6 @@ ved_include(struct req *preq, const char *src, const char *host,
 	preq->vcl = req->vcl;
 	req->vcl = NULL;
 
-	if (ecx->isgzip && req->l_crc) {
-		preq->crc = crc32_combine(
-		    preq->crc, req->crc, req->l_crc);
-		preq->l_crc += req->l_crc;
-	}
-
 	req->wrk = NULL;
 
 	THR_SetRequest(preq);
@@ -237,6 +235,7 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 		AZ(*priv);
 		ALLOC_OBJ(ecx, ECX_MAGIC);
 		AN(ecx);
+		ecx->preq = req;
 		*priv = ecx;
 		return (0);
 	}
@@ -273,9 +272,9 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 					/* Send out the gzip header */
 					retval = VDP_bytes(req, VDP_NULL,
 					    gzip_hdr, 10);
-					req->l_crc = 0;
 					req->gzip_resp = 1;
-					req->crc = crc32(0L, Z_NULL, 0);
+					ecx->l_crc = 0;
+					ecx->crc = crc32(0L, Z_NULL, 0);
 				}
 			}
 			ecx->state = 1;
@@ -299,9 +298,9 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 					icrc = vbe32dec(ecx->p);
 					ecx->p += 4;
 					if (req->gzip_resp) {
-						req->crc = crc32_combine(
-						    req->crc, icrc, l);
-						req->l_crc += l;
+						ecx->crc = crc32_combine(
+						    ecx->crc, icrc, l);
+						ecx->l_crc += l;
 					}
 				}
 				ecx->state = 3;
@@ -349,10 +348,10 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 				tailbuf[4] = 0xff;
 
 				/* Emit CRC32 */
-				vle32enc(tailbuf + 5, req->crc);
+				vle32enc(tailbuf + 5, ecx->crc);
 
 				/* MOD(2^32) length */
-				vle32enc(tailbuf + 9, req->l_crc);
+				vle32enc(tailbuf + 9, ecx->l_crc);
 
 				(void)VDP_bytes(req, VDP_NULL, tailbuf, 13);
 			}
@@ -419,13 +418,28 @@ ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv,
 	uint8_t buf1[5], buf2[5];
 	const uint8_t *p;
 	uint16_t lx;
+	struct ecx *ecx;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC);
+
 	(void)priv;
-	if (act == VDP_INIT || act == VDP_FINI)
+	if (act == VDP_INIT)
+		return (0);
+	if (act == VDP_FINI) {
+		*priv = NULL;
 		return (0);
+	}
+	if (l == 0)
+		return (VDP_bytes(req, act, pv, l));
+
 	p = pv;
 
+	if (ecx->isgzip) {
+		ecx->crc = crc32(ecx->crc, p, l);
+		ecx->l_crc += l;
+	}
+
 	lx = 65535;
 	buf1[0] = 0;
 	vle16enc(buf1 + 1, lx);
@@ -446,8 +460,6 @@ ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv,
 		}
 		if (VDP_bytes(req, VDP_NULL, p, lx))
 			return (-1);
-		req->crc = crc32(req->crc, p, lx);
-		req->l_crc += lx;
 		l -= lx;
 		p += lx;
 	}
@@ -475,9 +487,11 @@ ved_stripgzip(struct req *req)
 	void *oi;
 	void *sp;
 	ssize_t sl, ll, dl;
+	struct ecx *ecx;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+	CAST_OBJ_NOTNULL(ecx, req->transport_priv, ECX_MAGIC);
 
 	AN(ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED));
 
@@ -645,8 +659,9 @@ ved_stripgzip(struct req *req)
 
 	icrc = vle32dec(tailbuf);
 	ilen = vle32dec(tailbuf + 4);
-	req->crc = crc32_combine(req->crc, icrc, ilen);
-	req->l_crc += ilen;
+
+	ecx->crc = crc32_combine(ecx->crc, icrc, ilen);
+	ecx->l_crc += ilen;
 }
 
 /*--------------------------------------------------------------------*/
@@ -675,18 +690,18 @@ static void __match_proto__(vtr_deliver_f)
 VED_Deliver(struct req *req, struct busyobj *bo)
 {
 	int i;
-	struct req *preq;
+	struct ecx *ecx;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
-	CAST_OBJ_NOTNULL(preq, req->transport_priv, REQ_MAGIC);
+	CAST_OBJ_NOTNULL(ecx, req->transport_priv, ECX_MAGIC);
 
 	req->res_mode |= RES_ESI_CHILD;
 	i = ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED);
 	if (req->gzip_resp && i && !(req->res_mode & RES_ESI)) {
-		VDP_push(req, ved_vdp_bytes, preq, 1);
+		VDP_push(req, ved_vdp_bytes, ecx->preq, 1);
 
 		if (bo != NULL)
 			VBO_waitstate(bo, BOS_FINISHED);
@@ -694,9 +709,9 @@ VED_Deliver(struct req *req, struct busyobj *bo)
 		(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
 	} else {
 		if (req->gzip_resp && !i)
-			VDP_push(req, ved_pretend_gzip, NULL, 1);
+			VDP_push(req, ved_pretend_gzip, ecx, 1);
 
-		VDP_push(req, ved_vdp_bytes, preq, 1);
+		VDP_push(req, ved_vdp_bytes, ecx->preq, 1);
 		(void)VDP_DeliverObj(req);
 	}
 	VDP_close(req);



From phk at FreeBSD.org  Thu May 14 21:13:50 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 14 May 2015 23:13:50 +0200
Subject: [master] b0bea23 Further simplicfication of ESI delivery
Message-ID: <E1Yt0Ry-0002o5-MN@project.varnish-software.com>


commit b0bea23e64342dabfa20b2a3be787a0f140d53ff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 21:13:29 2015 +0000

    Further simplicfication of ESI delivery

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 5bb360a..c8865c2 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -144,7 +144,6 @@ ved_include(struct req *preq, const char *src, const char *host,
 	req->t_req = preq->t_req;
 	assert(isnan(req->t_first));
 	assert(isnan(req->t_prev));
-	req->gzip_resp = ecx->isgzip;
 
 	INIT_OBJ(&xp, TRANSPORT_MAGIC);
 	xp.deliver = VED_Deliver;
@@ -266,13 +265,11 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 				 * Only the top level document gets to
 				 * decide this.
 				 */
-				req->gzip_resp = 0;
 				if (ecx->isgzip) {
 					assert(sizeof gzip_hdr == 10);
 					/* Send out the gzip header */
 					retval = VDP_bytes(req, VDP_NULL,
 					    gzip_hdr, 10);
-					req->gzip_resp = 1;
 					ecx->l_crc = 0;
 					ecx->crc = crc32(0L, Z_NULL, 0);
 				}
@@ -297,7 +294,7 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 					l = ved_decode_len(&ecx->p);
 					icrc = vbe32dec(ecx->p);
 					ecx->p += 4;
-					if (req->gzip_resp) {
+					if (ecx->isgzip) {
 						ecx->crc = crc32_combine(
 						    ecx->crc, icrc, l);
 						ecx->l_crc += l;
@@ -336,7 +333,7 @@ VDP_ESI(struct req *req, enum vdp_action act, void **priv,
 			}
 			break;
 		case 2:
-			if (req->gzip_resp && req->esi_level == 0) {
+			if (ecx->isgzip && req->esi_level == 0) {
 				/*
 				 * We are bytealigned here, so simply emit
 				 * a gzip literal block with finish bit set.
@@ -431,7 +428,7 @@ ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv,
 		return (0);
 	}
 	if (l == 0)
-		return (VDP_bytes(req, act, pv, l));
+		return (VDP_bytes(ecx->preq, act, pv, l));
 
 	p = pv;
 
@@ -448,23 +445,23 @@ ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv,
 	while (l > 0) {
 		if (l >= 65535) {
 			lx = 65535;
-			if (VDP_bytes(req, VDP_NULL, buf1, sizeof buf1))
+			if (VDP_bytes(ecx->preq, VDP_NULL, buf1, sizeof buf1))
 				return (-1);
 		} else {
 			lx = (uint16_t)l;
 			buf2[0] = 0;
 			vle16enc(buf2 + 1, lx);
 			vle16enc(buf2 + 3, ~lx);
-			if (VDP_bytes(req, VDP_NULL, buf2, sizeof buf2))
+			if (VDP_bytes(ecx->preq, VDP_NULL, buf2, sizeof buf2))
 				return (-1);
 		}
-		if (VDP_bytes(req, VDP_NULL, p, lx))
+		if (VDP_bytes(ecx->preq, VDP_NULL, p, lx))
 			return (-1);
 		l -= lx;
 		p += lx;
 	}
 	/* buf2 is local, have to flush */
-	return (VDP_bytes(req, VDP_FLUSH, NULL, 0));
+	return (VDP_bytes(ecx->preq, VDP_FLUSH, NULL, 0));
 }
 
 /*---------------------------------------------------------------------
@@ -548,7 +545,7 @@ ved_stripgzip(struct req *req)
 			if (dl > 0) {
 				if (dl > sl)
 					dl = sl;
-				if (VDP_bytes(req, VDP_NULL, pp, dl))
+				if (VDP_bytes(ecx->preq, VDP_NULL, pp, dl))
 					break;
 				ll += dl;
 				sl -= dl;
@@ -559,7 +556,7 @@ ved_stripgzip(struct req *req)
 			/* Remove the "LAST" bit */
 			dbits[0] = *pp;
 			dbits[0] &= ~(1U << (last & 7));
-			if (VDP_bytes(req, VDP_NULL, dbits, 1))
+			if (VDP_bytes(ecx->preq, VDP_NULL, dbits, 1))
 				break;
 			ll++;
 			sl--;
@@ -571,7 +568,7 @@ ved_stripgzip(struct req *req)
 			if (dl > 0) {
 				if (dl > sl)
 					dl = sl;
-				if (VDP_bytes(req, VDP_NULL, pp, dl))
+				if (VDP_bytes(ecx->preq, VDP_NULL, pp, dl))
 					break;
 				ll += dl;
 				sl -= dl;
@@ -633,7 +630,7 @@ ved_stripgzip(struct req *req)
 			default:
 				WRONG("compiler must be broken");
 			}
-			if (VDP_bytes(req, VDP_NULL, dbits + 1, lpad))
+			if (VDP_bytes(ecx->preq, VDP_NULL, dbits + 1, lpad))
 				break;
 		}
 		if (sl > 0) {
@@ -656,6 +653,7 @@ ved_stripgzip(struct req *req)
 		}
 	} while (ois == OIS_DATA || ois == OIS_STREAM);
 	ObjIterEnd(req->objcore, &oi);
+	(void)VDP_bytes(ecx->preq, VDP_FLUSH, NULL, 0);
 
 	icrc = vle32dec(tailbuf);
 	ilen = vle32dec(tailbuf + 4);
@@ -700,18 +698,15 @@ VED_Deliver(struct req *req, struct busyobj *bo)
 
 	req->res_mode |= RES_ESI_CHILD;
 	i = ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED);
-	if (req->gzip_resp && i && !(req->res_mode & RES_ESI)) {
-		VDP_push(req, ved_vdp_bytes, ecx->preq, 1);
-
+	if (ecx->isgzip && i && !(req->res_mode & RES_ESI)) {
 		if (bo != NULL)
 			VBO_waitstate(bo, BOS_FINISHED);
 		ved_stripgzip(req);
-		(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
 	} else {
-		if (req->gzip_resp && !i)
+		if (ecx->isgzip && !i)
 			VDP_push(req, ved_pretend_gzip, ecx, 1);
-
-		VDP_push(req, ved_vdp_bytes, ecx->preq, 1);
+		else
+			VDP_push(req, ved_vdp_bytes, ecx->preq, 1);
 		(void)VDP_DeliverObj(req);
 	}
 	VDP_close(req);
diff --git a/include/tbl/req_flags.h b/include/tbl/req_flags.h
index 0571006..389cc51 100644
--- a/include/tbl/req_flags.h
+++ b/include/tbl/req_flags.h
@@ -35,5 +35,4 @@ REQ_FLAG(hash_ignore_busy,	1, 1, "")
 REQ_FLAG(hash_always_miss,	1, 1, "")
 REQ_FLAG(is_hit,		0, 0, "")
 REQ_FLAG(wantbody,		0, 0, "")
-REQ_FLAG(gzip_resp,		0, 0, "")
 /*lint -restore */



From phk at FreeBSD.org  Thu May 14 22:10:09 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 15 May 2015 00:10:09 +0200
Subject: [master] de91486 Add explanation for not streaming
Message-ID: <E1Yt1KT-00005R-Um@project.varnish-software.com>


commit de9148622abcc226725a36291ac8f5dd88266c09
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 21:45:56 2015 +0000

    Add explanation for not streaming

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index c8865c2..9aeba38 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -466,10 +466,25 @@ ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv,
 
 /*---------------------------------------------------------------------
  * Include an object in a gzip'ed ESI object delivery
+ *
+ * This is not written as a VDP (yet) because it relies on the
+ * OA_GZIPBITS which only becomes available when the input side
+ * has fully digested the object and located the magic bit positions.
+ *
+ * We can improve this two ways.
+ *
+ * One is to run a gunzip instance here, to find the stopbit ourselves,
+ * but that would be double work, in particular when passing a gziped
+ * object, where we would have two null-gunzips.
+ *
+ * The other is to have the input side guarantee that OA_GZIPBITS::stopbit
+ * always is committed before the chunk of data containing it.  We would
+ * be required to poll OA_GZIPBITS on every chunk presented, but that is
+ * much cheaper than running a gunzip instance.
  */
 
 static void
-ved_stripgzip(struct req *req)
+ved_stripgzip(struct req *req, struct busyobj *bo)
 {
 	ssize_t start, last, stop, lpad;
 	ssize_t l;
@@ -490,6 +505,9 @@ ved_stripgzip(struct req *req)
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 	CAST_OBJ_NOTNULL(ecx, req->transport_priv, ECX_MAGIC);
 
+	if (bo != NULL)
+		VBO_waitstate(bo, BOS_FINISHED);
+
 	AN(ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED));
 
 	/*
@@ -699,9 +717,7 @@ VED_Deliver(struct req *req, struct busyobj *bo)
 	req->res_mode |= RES_ESI_CHILD;
 	i = ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED);
 	if (ecx->isgzip && i && !(req->res_mode & RES_ESI)) {
-		if (bo != NULL)
-			VBO_waitstate(bo, BOS_FINISHED);
-		ved_stripgzip(req);
+		ved_stripgzip(req, bo);
 	} else {
 		if (ecx->isgzip && !i)
 			VDP_push(req, ved_pretend_gzip, ecx, 1);



From phk at FreeBSD.org  Thu May 14 22:10:09 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 15 May 2015 00:10:09 +0200
Subject: [master] 3bd29b4 Be much more cautions about create C-L headers from
 OA_GZIPBITS
Message-ID: <E1Yt1KT-00005X-W3@project.varnish-software.com>


commit 3bd29b4d0335d856831ba0c60623a8f4f7ccc8a9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 22:09:46 2015 +0000

    Be much more cautions about create C-L headers from OA_GZIPBITS

diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 3bc1ed0..d0b1ee0 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -293,20 +293,24 @@ VDP_gunzip(struct req *req, enum vdp_action act, void **priv,
 		VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
 		*priv = vg;
 
-		p = ObjGetattr(req->wrk, req->objcore, OA_GZIPBITS, &dl);
-		if (p != NULL && dl == 32) {
-			u = vbe64dec(p + 24);
-			/*
-			 * If the size is non-zero, and we are the top
-			 * VDP, we know what size the output will be.
-			 */
-			if (u != 0 &&
-			    VTAILQ_FIRST(&req->vdp)->func == VDP_gunzip)
-				req->resp_len = u;
-			else
-				req->resp_len = -1;
-		}
 		http_Unset(req->resp, H_Content_Encoding);
+
+		req->resp_len = -1;
+		if (req->objcore->busyobj != NULL)
+			return (0); /* Incomplete, no idea about length */
+
+		p = ObjGetattr(req->wrk, req->objcore, OA_GZIPBITS, &dl);
+		if (p == NULL || dl != 32)
+			return (0); /* No OA_GZIPBITS yet */
+
+		u = vbe64dec(p + 24);
+		/*
+		 * If the size is non-zero AND we are the top
+		 * VDP (ie: no ESI), we know what size the output will be.
+		 */
+		if (u != 0 && VTAILQ_FIRST(&req->vdp)->func == VDP_gunzip)
+			req->resp_len = u;
+
 		return (0);
 	}
 



From phk at FreeBSD.org  Thu May 14 23:10:48 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 15 May 2015 01:10:48 +0200
Subject: [master] 6eb3f1e Update the magic gzip bit positions whenever we spot
 them changing.
Message-ID: <E1Yt2HA-0006Nq-Kn@project.varnish-software.com>


commit 6eb3f1eb75e8cbaf5f3bec5c8eb5ae0b563c6ec5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 14 22:44:33 2015 +0000

    Update the magic gzip bit positions whenever we spot them changing.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index f9e5326..cad336e 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -822,7 +822,14 @@ enum vgzret_e VGZ_Gzip(struct vgz *, const void **, ssize_t *len,
     enum vgz_flag);
 // enum vgzret_e VGZ_Gunzip(struct vgz *, const void **, ssize_t *len);
 enum vgzret_e VGZ_Destroy(struct vgz **);
-void VGZ_UpdateObj(const struct vfp_ctx *, const struct vgz*, int input);
+
+enum vgz_ua_e {
+	VUA_UPDATE,		// Update start/stop/last bits if changed
+	VUA_END_GZIP,		// Record uncompressed size
+	VUA_END_GUNZIP,		// Record uncompressed size
+};
+
+void VGZ_UpdateObj(const struct vfp_ctx *, struct vgz*, enum vgz_ua_e);
 
 /* cache_http.c */
 unsigned HTTP_estimate(unsigned nhttp);
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 0765302..558ee15 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -94,6 +94,7 @@ vfp_vep_callback(struct vfp_ctx *vc, void *priv, ssize_t l, enum vgz_flag flg)
 		}
 		VGZ_Obuf(vef->vgz, ptr, dl);
 		i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
+		VGZ_UpdateObj(vc, vef->vgz, VUA_UPDATE);
 		vef->tot += dl;
 		VBO_extend(vc->bo, dl);
 	} while (i != VGZ_ERROR &&
@@ -131,7 +132,7 @@ vfp_esi_end(struct vfp_ctx *vc, struct vef_priv *vef,
 	}
 
 	if (vef->vgz != NULL) {
-		VGZ_UpdateObj(vc, vef->vgz, 1);
+		VGZ_UpdateObj(vc, vef->vgz, VUA_END_GZIP);
 		if (VGZ_Destroy(&vef->vgz) != VGZ_END)
 			retval = VFP_Error(vc,
 			    "ESI+Gzip Failed at the very end");
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index d0b1ee0..272a284 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -59,6 +59,8 @@ struct vgz {
 	ssize_t			m_sz;
 	ssize_t			m_len;
 
+	intmax_t		bits;
+
 	z_stream		vz;
 };
 
@@ -348,19 +350,24 @@ VDP_gunzip(struct req *req, enum vdp_action act, void **priv,
 /*--------------------------------------------------------------------*/
 
 void
-VGZ_UpdateObj(const struct vfp_ctx *vc, const struct vgz *vg, int input)
+VGZ_UpdateObj(const struct vfp_ctx *vc, struct vgz *vg, enum vgz_ua_e e)
 {
 	char *p;
+	intmax_t ii;
 
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	ii = vg->vz.start_bit + vg->vz.last_bit + vg->vz.stop_bit;
+	if (e == VUA_UPDATE && ii == vg->bits)
+		return;
+	vg->bits = ii;
 	p = ObjSetattr(vc->wrk, vc->oc, OA_GZIPBITS, 32, NULL);
 	AN(p);
 	vbe64enc(p, vg->vz.start_bit);
 	vbe64enc(p + 8, vg->vz.last_bit);
 	vbe64enc(p + 16, vg->vz.stop_bit);
-	if (input)
+	if (e == VUA_END_GZIP)
 		vbe64enc(p + 24, vg->vz.total_in);
-	else
+	if (e == VUA_END_GUNZIP)
 		vbe64enc(p + 24, vg->vz.total_out);
 }
 
@@ -553,6 +560,7 @@ vfp_gzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
 			if (vr < VGZ_OK)
 				return (VFP_Error(vc, "Gzip failed"));
 			if (dl > 0) {
+				VGZ_UpdateObj(vc, vg, VUA_UPDATE);
 				*lp = dl;
 				assert(dp == p);
 				return (VFP_OK);
@@ -563,7 +571,7 @@ vfp_gzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
 
 	if (vr != VGZ_END)
 		return (VFP_Error(vc, "Gzip failed"));
-	VGZ_UpdateObj(vc, vg, 1);
+	VGZ_UpdateObj(vc, vg, VUA_END_GZIP);
 	return (VFP_END);
 }
 
@@ -605,10 +613,11 @@ vfp_testgunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
 				    "Invalid Gzip data: %s", vg->vz.msg));
 		} while (!VGZ_IbufEmpty(vg));
 	}
+	VGZ_UpdateObj(vc, vg, VUA_UPDATE);
 	if (vp == VFP_END) {
 		if (vr != VGZ_END)
 			return (VFP_Error(vc, "tGunzip failed"));
-		VGZ_UpdateObj(vc, vg, 0);
+		VGZ_UpdateObj(vc, vg, VUA_END_GUNZIP);
 	}
 	return (vp);
 }



From phk at FreeBSD.org  Fri May 15 13:59:36 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 15 May 2015 15:59:36 +0200
Subject: [master] 2679bb6 Allow return(abandon) in vcl_backend_error{}
Message-ID: <E1YtG9I-00059v-3W@project.varnish-software.com>


commit 2679bb6db2f30b29f31da629927c7e0b243ea7d3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 15 13:58:43 2015 +0000

    Allow return(abandon) in vcl_backend_error{}
    
    Submitted by:	fgs

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index b29eb25..b52354a 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -800,14 +800,16 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 
 	AZ(VSB_finish(synth_body));
 
-	if (wrk->handling == VCL_RET_RETRY) {
+	if (wrk->handling == VCL_RET_RETRY ||
+	    wrk->handling == VCL_RET_ABANDON) {
 		VSB_delete(synth_body);
 
 		bo->doclose = SC_RESP_CLOSE;
 		if (bo->director_state != DIR_S_NULL)
 			VDI_Finish(bo->wrk, bo);
 
-		if (bo->retries++ < cache_param->max_retries)
+		if (wrk->handling == VCL_RET_RETRY &&
+		    bo->retries++ < cache_param->max_retries)
 			return (F_STP_RETRY);
 
 		return (F_STP_FAIL);
diff --git a/bin/varnishtest/tests/b00038.vtc b/bin/varnishtest/tests/b00038.vtc
index 963dc48..b963b4f 100644
--- a/bin/varnishtest/tests/b00038.vtc
+++ b/bin/varnishtest/tests/b00038.vtc
@@ -1,18 +1,37 @@
-varnishtest "vcl_backend_fetch abandon"
+varnishtest "Test abandon from vcl_backend_xxx"
 
 server s1 {
 	rxreq
+	expect req.url == "/bar"
 	txresp
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -arg "-pfirst_byte_timeout=0.1" -vcl+backend {
 	sub vcl_backend_fetch {
+		if (bereq.url == "/foo") {
+			return (abandon);
+		}
+	}
+	sub vcl_backend_response {
+		if (bereq.url == "/bar") {
+			return (abandon);
+		}
+	}
+	sub vcl_backend_error {
 		return (abandon);
 	}
 } -start
 
 client c1 {
-	txreq
+	txreq -url /foo
+	rxresp
+	expect resp.status == 503
+
+	txreq -url /bar
+	rxresp
+	expect resp.status == 503
+
+	txreq -url /baz
 	rxresp
 	expect resp.status == 503
 } -run
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 27b3e17..f342012 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -132,7 +132,7 @@ returns =(
 	),
 	('backend_error',
 		"B",
-		('deliver', 'retry')
+		('deliver', 'retry', 'abandon')
 	),
 
 	###############################################################



From lkarsten at varnish-software.com  Fri May 15 14:14:12 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 15 May 2015 16:14:12 +0200
Subject: [master] 2e964eb Document new TCP endpoint variables.
Message-ID: <E1YtGNQ-0006wc-Gf@project.varnish-software.com>


commit 2e964eb2143fdd5bddf8ff7eeb39b4d486eeec5f
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri May 15 16:14:00 2015 +0200

    Document new TCP endpoint variables.

diff --git a/doc/sphinx/whats-new/changes.rst b/doc/sphinx/whats-new/changes.rst
index c5d181b..e461f4c 100644
--- a/doc/sphinx/whats-new/changes.rst
+++ b/doc/sphinx/whats-new/changes.rst
@@ -78,6 +78,19 @@ The ``-a`` startup argument syntax has been expanded to allow for this::
 Both PROXY1 and PROXY2 protocols are supported on the resulting listening
 socket.
 
+For connections coming in over a PROXY socket, ``client.ip`` and
+``server.ip`` will contain the adresses given to Varnish in the PROXY
+header/preamble. (the "real" client IP.)
+
+The new VCL variables ``remote.ip`` and ``local.ip`` contains the local TCP
+connection endpoints. On non-PROXY connections these will be identical to
+``client.ip`` and ``server.ip``.
+
+An expected pattern following this is `if (std.port(local.ip) == 80) { }` in
+``vcl_recv`` to see if traffic came in over the HTTP listening socket. (so a client
+redirect to HTTPS can be served.)
+
+
 
 VMOD backends
 =============



From phk at FreeBSD.org  Fri May 15 21:41:22 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 15 May 2015 23:41:22 +0200
Subject: [master] eb8f35b Close a race condition in the backend/tcp_pools
 interface.
Message-ID: <E1YtNMA-00089g-61@project.varnish-software.com>


commit eb8f35bd79206ff489b895ffb2dc3fb07cabfce5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 15 21:40:59 2015 +0000

    Close a race condition in the backend/tcp_pools interface.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 9aac126..858ade7 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -135,7 +135,7 @@ vbe_dir_getfd(const struct director *d, struct busyobj *bo)
 	VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s",
 	    vc->fd, bp->display_name, abuf2, pbuf2, abuf1, pbuf1);
 
-	vc->backend->vsc->req++;
+	bp->vsc->req++;
 	INIT_OBJ(bo->htc, HTTP_CONN_MAGIC);
 	bo->htc->vbc = vc;
 	bo->htc->fd = vc->fd;
@@ -167,11 +167,12 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
 	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
 	if (bo->htc->vbc == NULL)
 		return;
-	bp = bo->htc->vbc->backend;
+	bo->htc->vbc->backend = NULL;
 	if (bo->doclose != SC_NULL) {
 		VSLb(bo->vsl, SLT_BackendClose, "%d %s", bo->htc->vbc->fd,
 		    bp->display_name);
@@ -187,8 +188,8 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 #define ACCT(foo)	bp->vsc->foo += bo->acct.foo;
 #include "tbl/acct_fields_bereq.h"
 #undef ACCT
-	Lck_Unlock(&bp->mtx);
 	bo->htc->vbc = NULL;
+	Lck_Unlock(&bp->mtx);
 	bo->htc = NULL;
 }
 
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 48dffa4..38a979f 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -100,6 +100,7 @@ struct vbc {
 	uint8_t			in_waiter;
 	uint8_t			have_been_in_waiter;
 	struct waited		waited[1];
+	struct tcp_pool		*tcp_pool;
 
 	struct backend		*backend;
 };
diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index 5b73e57..518870e 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -85,7 +85,8 @@ tcp_handle(struct waited *w, enum wait_event ev, double now)
 	CAST_OBJ_NOTNULL(vbc, w->ptr, VBC_MAGIC);
 	(void)ev;
 	(void)now;
-	tp = vbc->backend->tcp_pool;			// NB: Incestous
+	CHECK_OBJ_NOTNULL(vbc->tcp_pool, TCP_POOL_MAGIC);
+	tp = vbc->tcp_pool;
 
 	Lck_Lock(&tp->mtx);
 	VSL(SLT_Debug, 0,
@@ -388,6 +389,7 @@ VBT_Get(struct tcp_pool *tp, double tmo)
 		tp->n_conn--;
 		VSC_C_main->backend_reuse += 1;
 		vbc->state = VBC_STATE_USED;
+		assert(vbc->tcp_pool == tp);
 	}
 	tp->n_used++;			// Opening mostly works
 	Lck_Unlock(&tp->mtx);
@@ -399,6 +401,7 @@ VBT_Get(struct tcp_pool *tp, double tmo)
 	AN(vbc);
 	INIT_OBJ(vbc->waited, WAITED_MAGIC);
 	vbc->state = VBC_STATE_USED;
+	vbc->tcp_pool = tp;
 	vbc->fd = VBT_Open(tp, tmo, &vbc->addr);
 	if (vbc->fd < 0)
 		FREE_OBJ(vbc);



From phk at FreeBSD.org  Sat May 16 08:38:08 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 16 May 2015 10:38:08 +0200
Subject: [master] c2cb5e7 Add read-only access to the top level request in an
 ESI tree.
Message-ID: <E1YtXbk-0001KD-0B@project.varnish-software.com>


commit c2cb5e71ac123fce9d4cd94ba09f6b34f6fa1828
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 16 08:36:51 2015 +0000

    Add read-only access to the top level request in an ESI tree.
    
    Patch from daghf (with minor changes)

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index a453cf3..18631f8 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -475,6 +475,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 		vsl = req->vsl;
 		ctx.vcl = req->vcl;
 		ctx.http_req = req->http;
+		ctx.http_req_top = req->top->http;
 		ctx.http_resp = req->resp;
 		ctx.req = req;
 		ctx.now = req->t_prev;
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 5a12c63..9b4aaff 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -96,6 +96,9 @@ VRT_selecthttp(VRT_CTX, enum gethdr_e where)
 	case HDR_REQ:
 		hp = ctx->http_req;
 		break;
+	case HDR_REQ_TOP:
+		hp = ctx->http_req_top;
+		break;
 	case HDR_BEREQ:
 		hp = ctx->http_bereq;
 		break;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 5ae624e..cc302c5 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -119,6 +119,10 @@ VRT_HDR_LR(req,    method,	HTTP_HDR_METHOD)
 VRT_HDR_LR(req,    url,		HTTP_HDR_URL)
 VRT_HDR_LR(req,    proto,	HTTP_HDR_PROTO)
 
+VRT_HDR_R(req_top,    method,	HTTP_HDR_METHOD)
+VRT_HDR_R(req_top,    url,	HTTP_HDR_URL)
+VRT_HDR_R(req_top,    proto,	HTTP_HDR_PROTO)
+
 VRT_HDR_LR(resp,   proto,	HTTP_HDR_PROTO)
 VRT_HDR_LR(resp,   reason,	HTTP_HDR_REASON)
 VRT_STATUS_L(resp)
diff --git a/bin/varnishtest/tests/e00030.vtc b/bin/varnishtest/tests/e00030.vtc
new file mode 100644
index 0000000..7779f98
--- /dev/null
+++ b/bin/varnishtest/tests/e00030.vtc
@@ -0,0 +1,76 @@
+varnishtest "Test req_top.* in an ESI context"
+
+varnish v1 -errvcl {Variable 'req_top.url' is read only.} {
+	sub vcl_recv {
+		set req_top.url = "/foo";
+	}
+}
+
+server s1 {
+	rxreq
+	expect req.http.top-url == "/"
+	expect req.http.top-method == "GET"
+	expect req.http.top-proto == "HTTP/1.1"
+	expect req.http.top-foo == "bar"
+	txresp -body {
+		<html>
+		Before include
+		<esi:include src="/a"/>
+		<esi:include src="/b"/>
+		After include
+		</html>
+	}
+
+	rxreq
+	expect req.url == "/a1"
+	expect req.http.top-url == "/"
+	expect req.http.top-method == "GET"
+	expect req.http.top-proto == "HTTP/1.1"
+	expect req.http.top-foo == "bar"
+	txresp -body {
+		Included file
+		<esi:include src="/c"/>
+	}
+
+	rxreq
+	expect req.http.top-url == "/"
+	expect req.http.top-method == "GET"
+	expect req.http.top-proto == "HTTP/1.1"
+	expect req.http.top-foo == "bar"
+	expect req.url == "/b1"
+	txresp
+
+	rxreq
+	expect req.http.top-url == "/"
+	expect req.http.top-method == "GET"
+	expect req.http.top-proto == "HTTP/1.1"
+	expect req.http.top-foo == "bar"
+	expect req.url == "/c2"
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.esi_level > 0) {
+			set req.url = req.url + req.esi_level;
+		} else {
+			set req.http.foo = "bar";
+		}
+
+		set req.http.top-url = req_top.url;
+		set req.http.top-method = req_top.method;
+		set req.http.top-proto = req_top.proto;
+		set req.http.top-foo = req_top.http.foo;
+	}
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
+
+varnish v1 -expect esi_errors == 0
diff --git a/include/vrt.h b/include/vrt.h
index 785dd8c..dca5902 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -100,6 +100,7 @@ struct vrt_ctx {
 
 	struct req			*req;
 	struct http			*http_req;
+	struct http			*http_req_top;
 	struct http			*http_resp;
 
 	struct busyobj			*bo;
@@ -136,7 +137,8 @@ struct vmod_data {
 
 /***********************************************************************/
 
-enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_BEREQ, HDR_BERESP };
+enum gethdr_e { HDR_REQ, HDR_REQ_TOP, HDR_RESP, HDR_OBJ, HDR_BEREQ,
+		HDR_BERESP };
 
 struct gethdr_s {
 	enum gethdr_e	where;
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index f342012..f695dc7 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -317,6 +317,41 @@ sp_variables = [
 		always (re)fetch from the backend.
 		"""
 	),
+	('req_top.method',
+		'STRING',
+		( 'client',),
+		(), """
+		The request method of the top-level request in a tree
+		of ESI requests. (e.g. "GET", "HEAD").
+		Identical to req.method in non-ESI requests.
+		"""
+	),
+	('req_top.url',
+		'STRING',
+		( 'client',),
+		(), """
+		The requested URL of the top-level request in a tree
+		of ESI requests.
+		Identical to req.url in non-ESI requests.
+		"""
+	),
+	('req_top.http.',
+		'HEADER',
+		( 'client',),
+		(), """
+		HTTP headers of the top-level request in a tree of ESI requests.
+		Identical to req.http. in non-ESI requests.
+		"""
+	),
+	('req_top.proto',
+		'STRING',
+		( 'client',),
+		(), """
+		HTTP protocol version of the top-level request in a tree of
+		ESI requests.
+		Identical to req.proto in non-ESI requests.
+		"""
+	),
 	('bereq',
 		'HTTP',
 		( 'backend',),



From phk at FreeBSD.org  Sat May 16 09:05:25 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 16 May 2015 11:05:25 +0200
Subject: [master] d7b9296 Certain failure scenarios,
 for instance PROXY protocol syntax errors,
 are handled so fast that there may be no numerical difference between the
 start and end session timestamps.
Message-ID: <E1YtY29-00039j-JL@project.varnish-software.com>


commit d7b9296012f6a5d41c48198ab2ea991a173c3b2c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 16 08:59:57 2015 +0000

    Certain failure scenarios, for instance PROXY protocol syntax errors,
    are handled so fast that there may be no numerical difference between
    the start and end session timestamps.
    
    This is a consequence of our decision to use C type double (ieee 64bit FP)
    for timestamps, at the current distance from POSIX epoch, 1.5 billion
    seconds, the resolution is 238 nanoseconds.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 1f6a8ef..6153f84 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -560,7 +560,7 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
 	if (reason == SC_NULL)
 		reason = (enum sess_close)-sp->fd;
 
-	assert(now > sp->t_open);
+	assert(now >= sp->t_open);
 	assert(VTAILQ_EMPTY(&sp->privs->privs));
 	VSL(SLT_SessClose, sp->vxid, "%s %.3f",
 	    sess_close_2str(reason, 0), now - sp->t_open);



From phk at FreeBSD.org  Sat May 16 09:42:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 16 May 2015 11:42:04 +0200
Subject: [master] 24d8af2 Improve stability of this test case
Message-ID: <E1YtYbc-0005AG-7X@project.varnish-software.com>


commit 24d8af28eaf122c0ed70b1a4a7e5d6fc78028d3f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 16 09:41:49 2015 +0000

    Improve stability of this test case

diff --git a/bin/varnishtest/tests/r01086.vtc b/bin/varnishtest/tests/r01086.vtc
index dc2e0b3..2dd6e2b 100644
--- a/bin/varnishtest/tests/r01086.vtc
+++ b/bin/varnishtest/tests/r01086.vtc
@@ -22,9 +22,8 @@ server s1 {
 
 	delay .2
 
-	chunked "FOOBAR"
-
 	non-fatal
+	chunked "FOOBAR"
 	chunkedlen 0
 
 	accept
@@ -51,10 +50,12 @@ server s1 {
 
 	delay .2
 
+	non-fatal
 	chunked "FOOBAR"
-
 	chunkedlen 0
 
+	accept
+
 } -start
 
 varnish v1 -vcl+backend {



From phk at FreeBSD.org  Sun May 17 08:28:22 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 17 May 2015 10:28:22 +0200
Subject: [master] 415c7d8 Improve EOF detection.
Message-ID: <E1Yttvq-0003qy-H3@project.varnish-software.com>


commit 415c7d8e824f6265ce08d6dcb8c6916a8f674099
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 17 08:27:09 2015 +0000

    Improve EOF detection.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 8464662..5a291dc 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1217,7 +1217,7 @@ cmd_http_expect_close(CMD_ARGS)
 		i = poll(fds, 1, hp->timeout);
 		if (i == 0)
 			vtc_log(vl, hp->fatal, "Expected close: timeout");
-		if (i != 1 || !(fds[0].revents & POLLIN))
+		if (i != 1 || !(fds[0].revents & (POLLIN|POLLERR)))
 			vtc_log(vl, hp->fatal,
 			    "Expected close: poll = %d, revents = 0x%x",
 			    i, fds[0].revents);



From phk at FreeBSD.org  Sun May 17 08:28:22 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 17 May 2015 10:28:22 +0200
Subject: [master] b2e9410 Improve stability of these two testcases.
Message-ID: <E1Yttvq-0003r4-ID@project.varnish-software.com>


commit b2e94101c293843125f48f6ad22a5c6d98b69c55
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 17 08:27:37 2015 +0000

    Improve stability of these two testcases.
    
    We have to restart the server instance when we have writes which
    might fail.

diff --git a/bin/varnishtest/tests/r00942.vtc b/bin/varnishtest/tests/r00942.vtc
index 164234d..0497413 100644
--- a/bin/varnishtest/tests/r00942.vtc
+++ b/bin/varnishtest/tests/r00942.vtc
@@ -1,6 +1,6 @@
 varnishtest "#942 junk after gzip from backend"
 
-server s1 -repeat 2 {
+server s1 {
 	rxreq
 	txresp -nolen \
 		-hdr "Content-Encoding: gzip" \
@@ -18,13 +18,9 @@ server s1 -repeat 2 {
 		sendhex "00000000"
 		sendhex "00000000"
 	send "\r\n"
-
 	chunked "FOOBAR"
-
 	non-fatal
 	chunkedlen 0
-	accept
-	fatal
 } -start
 
 varnish v1 \
@@ -46,6 +42,31 @@ client c1 {
 	expect resp.status == 503
 } -run
 
+server s1 -wait {
+	fatal
+	rxreq
+	txresp -nolen \
+		-hdr "Content-Encoding: gzip" \
+		-hdr "Connection: close" \
+		-hdr "Transfer-Encoding: Chunked"
+	send "14\r\n"
+		# An empty gzip file:
+		sendhex "1f8b"
+		sendhex "08"
+		sendhex "00"
+		sendhex "00000000"
+		sendhex "00"
+		sendhex "03"
+		sendhex "0300"
+		sendhex "00000000"
+		sendhex "00000000"
+	send "\r\n"
+	chunked "FOOBAR"
+	non-fatal
+	chunkedlen 0
+} -start
+
+
 client c1 {
 	txreq -url /2 -hdr "Foo: foo"
 	rxresp
diff --git a/bin/varnishtest/tests/r01086.vtc b/bin/varnishtest/tests/r01086.vtc
index 2dd6e2b..79c5128 100644
--- a/bin/varnishtest/tests/r01086.vtc
+++ b/bin/varnishtest/tests/r01086.vtc
@@ -22,11 +22,30 @@ server s1 {
 
 	delay .2
 
-	non-fatal
 	chunked "FOOBAR"
+	non-fatal
 	chunkedlen 0
+} -start
+
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		if (beresp.http.set-cookie == "BAR") {
+			set beresp.do_stream = false;
+		}
+	}
+} -start
+
+client c1 {
+	txreq -hdr "Cookie: FOO"
+	rxresphdrs
+	expect resp.status == 200
+	expect_close
+} -run
+
+delay .1
 
-	accept
+server s1 -wait {
 	fatal
 
 	# This one will not be streamed
@@ -50,32 +69,13 @@ server s1 {
 
 	delay .2
 
-	non-fatal
 	chunked "FOOBAR"
+	non-fatal
 	chunkedlen 0
 
-	accept
-
-} -start
-
-varnish v1 -vcl+backend {
-	sub vcl_backend_response {
-		if (beresp.http.set-cookie == "BAR") {
-			set beresp.do_stream = false;
-		}
-	}
 } -start
 
 client c1 {
-	txreq -hdr "Cookie: FOO"
-	rxresphdrs
-	expect resp.status == 200
-	expect_close
-} -run
-
-delay .1
-
-client c1 {
 	txreq -hdr "Cookie: BAR"
 	rxresp
 	expect resp.status == 503



From phk at FreeBSD.org  Sun May 17 08:56:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 17 May 2015 10:56:04 +0200
Subject: [master] f8dd42a Expand the SIGSEGV handler to also cover SIGBUS
Message-ID: <E1YtuMe-0007Mh-BS@project.varnish-software.com>


commit f8dd42a485fa0ba79f99733b3a20aa9d6f71319b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 17 08:55:44 2015 +0000

    Expand the SIGSEGV handler to also cover SIGBUS

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 9ce3aaa..450ec9c 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -377,6 +377,7 @@ mgt_launch_child(struct cli *cli)
 			sa.sa_sigaction = child_sigsegv_handler;
 			sa.sa_flags = SA_SIGINFO;
 			(void)sigaction(SIGSEGV, &sa, NULL);
+			(void)sigaction(SIGBUS, &sa, NULL);
 		}
 		(void)signal(SIGINT, SIG_DFL);
 		(void)signal(SIGTERM, SIG_DFL);
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index ceb6b35..ae69bee 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -505,7 +505,7 @@ struct parspec mgt_parspec[] = {
 	{ "sigsegv_handler", tweak_bool, &mgt_param.sigsegv_handler,
 		NULL, NULL,
 		"Install a signal handler which tries to dump debug "
-		"information on segmentation faults.",
+		"information on segmentation and buserror faults.",
 		MUST_RESTART,
 		"off", "bool" },
 	{ "vcl_dir", tweak_string, &mgt_vcl_dir,



From phk at FreeBSD.org  Sun May 17 09:40:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 17 May 2015 11:40:04 +0200
Subject: [master] b59bf49 Alignment may push the size up
Message-ID: <E1Ytv3E-0000aC-D5@project.varnish-software.com>


commit b59bf492538a496f7822cca466bfb7f9f85cfee2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 17 09:30:55 2015 +0000

    Alignment may push the size up

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 36668c1..4bec380 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -202,7 +202,7 @@ Pool_Task_Arg(struct worker *wrk, task_func_t *func,
 	Lck_Unlock(&pp->mtx);
 	AZ(wrk2->task.func);
 
-	assert(arg_len == WS_Reserve(wrk2->aws, arg_len));
+	assert(arg_len <= WS_Reserve(wrk2->aws, arg_len));
 	memcpy(wrk2->aws->f, arg, arg_len);
 	wrk2->task.func = func;
 	wrk2->task.priv = wrk2->aws->f;



From phk at FreeBSD.org  Sun May 17 09:40:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 17 May 2015 11:40:04 +0200
Subject: [master] b703b6b Rewrite for more robust handling of server write
 failures
Message-ID: <E1Ytv3E-0000ac-Ek@project.varnish-software.com>


commit b703b6b379058faae06721203c9b14236d090d90
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 17 09:39:40 2015 +0000

    Rewrite for more robust handling of server write failures

diff --git a/bin/varnishtest/tests/c00046.vtc b/bin/varnishtest/tests/c00046.vtc
index 3abea2f..b5baf91 100644
--- a/bin/varnishtest/tests/c00046.vtc
+++ b/bin/varnishtest/tests/c00046.vtc
@@ -3,10 +3,6 @@ varnishtest	"Object/LRU/Stevedores with hinting and body alloc failures"
 server s1 {
 	rxreq
 	txresp -bodylen 1000000
-	rxreq
-	txresp -bodylen 1000001
-	rxreq
-	txresp -bodylen 1000002
 } -start
 
 varnish v1 \
@@ -35,6 +31,12 @@ varnish v1 -expect SMA.s1.g_space > 1000000
 varnish v1 -expect SMA.s2.g_bytes == 0
 varnish v1 -expect SMA.s2.g_space > 1000000
 
+server s1 -wait {
+	rxreq
+	non-fatal
+	txresp -bodylen 1000001
+} -start
+
 client c1 {
 	txreq -url /bar
 	rxresp
@@ -51,6 +53,12 @@ varnish v1 -expect SMA.s1.g_space > 1000000
 varnish v1 -expect SMA.s2.g_bytes == 0
 varnish v1 -expect SMA.s2.g_space > 1000000
 
+server s1 -wait {
+	rxreq
+	# non-fatal
+	txresp -bodylen 1000002
+} -start
+
 client c1 {
 	txreq -url /foo
 	rxresp



From phk at FreeBSD.org  Sun May 17 10:29:48 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 17 May 2015 12:29:48 +0200
Subject: [master] 0c3eaeb Increase alignment to uintmax_t to cater for 32bit
 userlands on 64bit processors.
Message-ID: <E1YtvpM-0001iH-Lp@project.varnish-software.com>


commit 0c3eaeb1a48706af53f283f714e15aaac3ef9ee7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 17 10:29:15 2015 +0000

    Increase alignment to uintmax_t to cater for 32bit userlands on
    64bit processors.

diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index b268082..b7b6c25 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -130,7 +130,7 @@ void VSM_common_ageupdate(const struct vsm_sc *sc);
  * Pointer aligment magic
  */
 
-#define PALGN	    (sizeof(void *) - 1)	/* size of alignment */
+#define PALGN	    (sizeof(uintmax_t) - 1)	/* size of alignment */
 #define PAOK(p)	    (((uintptr_t)(p) & PALGN) == 0)	/* is aligned */
 #define PRNDDN(p)   ((uintptr_t)(p) & ~PALGN)		/* Round down */
 #define PRNDUP(p)   (((uintptr_t)(p) + PALGN) & ~PALGN)	/* Round up */



From phk at FreeBSD.org  Sun May 17 16:08:08 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 17 May 2015 18:08:08 +0200
Subject: [master] cd55aa7 Make the ban header a multiple of 8 bytes for
 alignment
Message-ID: <E1Yu16m-0006Tg-NG@project.varnish-software.com>


commit cd55aa7f462130406419c38897987778670882dd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 17 15:59:09 2015 +0000

    Make the ban header a multiple of 8 bytes for alignment

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 84f8aab..e3e65f3 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -80,7 +80,7 @@
 #define BANS_TIMESTAMP		0
 #define BANS_LENGTH		8
 #define BANS_FLAGS		12
-#define BANS_HEAD_LEN		13
+#define BANS_HEAD_LEN		16
 
 #define BANS_FLAG_REQ		(1<<0)
 #define BANS_FLAG_OBJ		(1<<1)
@@ -514,6 +514,7 @@ BAN_Insert(struct ban *b)
 		return (ban_ins_error(NULL));
 	}
 
+	memset(b->spec, 0, BANS_HEAD_LEN);
 	t0 = VTIM_real();
 	memcpy(b->spec + BANS_TIMESTAMP, &t0, sizeof t0);
 	b->spec[BANS_FLAGS] = b->flags & 0xff;
@@ -832,7 +833,7 @@ ban_evaluate(struct worker *wrk, const uint8_t *bs, struct objcore *oc,
 	const char *arg1;
 
 	be = bs + ban_len(bs);
-	bs += 13;
+	bs += BANS_HEAD_LEN;
 	while (bs < be) {
 		(*tests)++;
 		ban_iter(&bs, &bt);



From phk at FreeBSD.org  Sun May 17 16:08:08 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 17 May 2015 18:08:08 +0200
Subject: [master] f17bd99 Brute force align ban-strings,
 to see if we can make the sparc64 box happy.
Message-ID: <E1Yu16m-0006Um-OR@project.varnish-software.com>


commit f17bd99ef29d1c0cc52c8b4b1233a5cb43738089
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 17 16:07:30 2015 +0000

    Brute force align ban-strings, to see if we can make the sparc64 box happy.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index e3e65f3..2124c33 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -292,6 +292,9 @@ ban_add_lump(const struct ban *b, const void *p, uint32_t len)
 {
 	uint8_t buf[sizeof len];
 
+	buf[0] = 0xff;
+	while (VSB_len(b->vsb) & PALGN)
+		VSB_bcat(b->vsb, buf, 1);
 	vbe32enc(buf, len);
 	VSB_bcat(b->vsb, buf, sizeof buf);
 	VSB_bcat(b->vsb, p, len);
@@ -303,6 +306,8 @@ ban_get_lump(const uint8_t **bs)
 	const void *r;
 	unsigned ln;
 
+	while (**bs == 0xff)
+		*bs += 1;
 	ln = vbe32dec(*bs);
 	*bs += 4;
 	r = (const void*)*bs;



From phk at FreeBSD.org  Mon May 18 07:31:10 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 18 May 2015 09:31:10 +0200
Subject: [master] ab38b9b Fix a brain-o uncovered by running our test-cases
 thousands of times over the weekend:
Message-ID: <E1YuFW2-0005ii-0I@project.varnish-software.com>


commit ab38b9b02a997a8e3fac02d19e009fb1c37cc0d2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 18 07:30:10 2015 +0000

    Fix a brain-o uncovered by running our test-cases thousands of times
    over the weekend:
    
    Just because we have a bo doesn't mean we have a bo->beresp any more,
    find the C-L header in the req->resp.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 1f0e306..653a0cd 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -54,7 +54,7 @@ cnt_vdp(struct req *req, struct busyobj *bo)
 
 	req->res_mode = 0;
 	if (bo != NULL)
-		req->resp_len = http_GetContentLength(bo->beresp);
+		req->resp_len = http_GetContentLength(req->resp);
 	else
 		req->resp_len = ObjGetLen(req->wrk, req->objcore);
 



From martin at varnish-software.com  Mon May 18 07:57:19 2015
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 18 May 2015 09:57:19 +0200
Subject: [master] 0f3be9f Tweak l00002.vtc
Message-ID: <E1YuFvL-0001oR-Nu@project.varnish-software.com>


commit 0f3be9fa4361da24ed31e4414d82af45f824ad6d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon May 18 09:56:20 2015 +0200

    Tweak l00002.vtc
    
    This hopefully makes l00002.vtc less prone to error.

diff --git a/bin/varnishtest/tests/l00002.vtc b/bin/varnishtest/tests/l00002.vtc
index 72faa08..942c8b5 100644
--- a/bin/varnishtest/tests/l00002.vtc
+++ b/bin/varnishtest/tests/l00002.vtc
@@ -67,21 +67,17 @@ varnish v1 -vcl+backend {
 # HTTP/1.1 400 Bad Request\r\n	 26 bytes
 # \r\n				  2 bytes
 # Total:			 28 bytes
-logexpect l1 -v v1 -g request {
-	expect 0 1001	Begin	"^req .* rxreq"
+logexpect l1 -v v1 -g session {
+	expect * 1001	Begin	"^req .* rxreq"
 	expect * =	ReqAcct	"^38 4 42 87 1000 1087$"
 	expect 0 =	End
-	expect 0 1002	Begin	"^bereq "
-	expect * =	End
-	expect 0 1003	Begin	"^req .* rxreq"
+	expect * 1003	Begin	"^req .* rxreq"
 	expect * =	ReqAcct "^19 0 19 87 2000 2087$"
 	expect 0 =	End
-	expect 0 1004	Begin	"^bereq"
-	expect * =	End
-	expect 0 1005	Begin	"^req .* rxreq"
+	expect * 1005	Begin	"^req .* rxreq"
 	expect * =	ReqAcct "^19 0 19 87 2000 2087$"
 	expect 0 =	End
-	expect 0 1006	Begin	"^req .* rxreq"
+	expect * 1006	Begin	"^req .* rxreq"
 	expect * =	ReqAcct	"^7 0 7 28 0 28$"
 	expect 0 =	End
 } -start



From martin at varnish-software.com  Mon May 18 08:08:43 2015
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 18 May 2015 10:08:43 +0200
Subject: [master] 1de529d Tweak l00005.vtc to make it less error prone
Message-ID: <E1YuG6N-0003mj-1X@project.varnish-software.com>


commit 1de529dd47471ea06b077928a53061feaa6aa064
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon May 18 10:08:29 2015 +0200

    Tweak l00005.vtc to make it less error prone

diff --git a/bin/varnishtest/tests/l00005.vtc b/bin/varnishtest/tests/l00005.vtc
index e7472ff..273eaaa 100644
--- a/bin/varnishtest/tests/l00005.vtc
+++ b/bin/varnishtest/tests/l00005.vtc
@@ -44,17 +44,18 @@ varnish v1 -vcl+backend {
 # \r\n				  2 bytes
 # Total:			 12 bytes
 
-logexpect l1 -v v1 -g request {
-	expect 0 1001	Begin		"^req .* rxreq"
+logexpect l1 -v v1 -g session {
+	expect * 1001	Begin		"^req .* rxreq"
 	expect * =	End
-	expect 0 1002	Begin		"^bereq "
-	expect * =	BereqAcct	"^54 4 58 41 1000 1041$"
-	expect 0 =	End
 	expect 0 1003	Begin		"^req .* rxreq"
 	expect * =	End
+        expect 0 1002   Begin           "^bereq "
+        expect * =      BereqAcct       "^54 4 58 41 1000 1041$"
+        expect 0 =      End
 	expect 0 1004	Begin		"^bereq"
 	expect * =	BereqAcct	"^54 4 58 12 0 12$"
 	expect * =	End
+
 } -start
 
 # Request 1001



From nils.goroll at uplex.de  Mon May 18 08:46:51 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 18 May 2015 10:46:51 +0200
Subject: [master] 2b1ac1c Ref: https://tools.ietf.org/html/rfc7231#section-6.1
Message-ID: <E1YuGhH-0007Qr-6i@project.varnish-software.com>


commit 2b1ac1ca1950026fe824c0194d0591579e4dda5b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 18 10:45:59 2015 +0200

    Ref: https://tools.ietf.org/html/rfc7231#section-6.1
    
    Changes:
    
    * 302 Found
    * 307 Temporary Redirect
    
      Do not apply the default ttl, only set a ttl if Cache-Control or Expires are
    present.
      Responses with these status codes are not cacheable by default
    
    * 414 Request-URI Too Large
    
      Cacheable with "heuristic expiration"
    
    https://www.varnish-cache.org/patchwork/patch/251/

diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index d1ce206..d3ea762 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -111,15 +111,18 @@ RFC2616_Ttl(struct busyobj *bo, double now)
 	default:
 		expp->ttl = -1.;
 		break;
+	case 302: /* Moved Temporarily */
+	case 307: /* Temporary Redirect */
+		expp->ttl = -1.;
 	case 200: /* OK */
 	case 203: /* Non-Authoritative Information */
+	case 204: /* No Content */
 	case 300: /* Multiple Choices */
 	case 301: /* Moved Permanently */
-	case 302: /* Moved Temporarily */
-	case 304: /* Not Modified */
-	case 307: /* Temporary Redirect */
-	case 410: /* Gone */
+	case 304: /* Not Modified - handled like 200 */
 	case 404: /* Not Found */
+	case 410: /* Gone */
+	case 414: /* Request-URI Too Large */
 		/*
 		 * First find any relative specification from the backend
 		 * These take precedence according to RFC2616, 13.2.4
diff --git a/bin/varnishtest/tests/b00015.vtc b/bin/varnishtest/tests/b00015.vtc
index 029f782..e26b16d 100644
--- a/bin/varnishtest/tests/b00015.vtc
+++ b/bin/varnishtest/tests/b00015.vtc
@@ -32,7 +32,7 @@ varnish v1 -cliok "ban req.url ~ .*"
 
 server s1 {
 	rxreq
-	txresp -status 302
+	txresp -status 301
 } -start
 
 varnish v1 -vcl+backend {
@@ -45,7 +45,7 @@ varnish v1 -vcl+backend {
 client c1 {
 	txreq -url "/"
 	rxresp
-	expect resp.status == 302
+	expect resp.status == 301
 	expect resp.http.X-varnish == "1007"
 } -run
 
@@ -54,7 +54,7 @@ delay .1
 client c1 {
 	txreq -url "/"
 	rxresp
-	expect resp.status == 302
+	expect resp.status == 301
 	expect resp.http.X-varnish == "1010 1008"
 } -run
 



From nils.goroll at uplex.de  Mon May 18 16:57:08 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 18 May 2015 18:57:08 +0200
Subject: [master] be28935 Avoid passing a NULL z_stream msg to vsnprintf via
 VSLb and VFP_Error
Message-ID: <E1YuOLk-0008GG-W8@project.varnish-software.com>


commit be28935476db8df6ecbbd45dcd6e79a933020c5f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 18 18:47:46 2015 +0200

    Avoid passing a NULL z_stream msg to vsnprintf via VSLb and VFP_Error
    
    Seen on a solaris vintage edition (snv_111b) as SIGSEGV caused
    by strlen(NULL) in r01036.vtc and r01037.vtc: a z_stream msg
    can be NULL.

diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 272a284..156b66f 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -64,6 +64,13 @@ struct vgz {
 	z_stream		vz;
 };
 
+static inline z_const char *
+vgz_msg(const struct vgz *vg)
+{
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	return vg->vz.msg ? vg->vz.msg : "(null)";
+}
+
 /*--------------------------------------------------------------------
  * Set up a gunzip instance
  */
@@ -220,7 +227,7 @@ VGZ_Gunzip(struct vgz *vg, const void **pptr, ssize_t *plen)
 		return (VGZ_END);
 	if (i == Z_BUF_ERROR)
 		return (VGZ_STUCK);
-	VSLb(vg->vsl, SLT_Gzip, "Gunzip error: %d (%s)", i, vg->vz.msg);
+	VSLb(vg->vsl, SLT_Gzip, "Gunzip error: %d (%s)", i, vgz_msg(vg));
 	return (VGZ_ERROR);
 }
 
@@ -261,7 +268,7 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags)
 		return (VGZ_END);
 	if (i == Z_BUF_ERROR)
 		return (VGZ_STUCK);
-	VSLb(vg->vsl, SLT_Gzip, "Gzip error: %d (%s)", i, vg->vz.msg);
+	VSLb(vg->vsl, SLT_Gzip, "Gzip error: %d (%s)", i, vgz_msg(vg));
 	return (VGZ_ERROR);
 }
 
@@ -385,6 +392,7 @@ VGZ_Destroy(struct vgz **vgp)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	*vgp = NULL;
 
+	AN(vg->id);
 	VSLb(vg->vsl, SLT_Gzip, "%s %jd %jd %jd %jd %jd",
 	    vg->id,
 	    (intmax_t)vg->vz.total_in,
@@ -408,7 +416,7 @@ VGZ_Destroy(struct vgz **vgp)
 		vr = VGZ_STUCK;
 	else {
 		VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
-		    i, vg->vz.msg);
+		    i, vgz_msg(vg));
 		vr = VGZ_ERROR;
 	}
 	FREE_OBJ(vg);
@@ -505,7 +513,7 @@ vfp_gunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
 				return(VFP_Error(vc, "Junk after gzip data"));
 			if (vr < VGZ_OK)
 				return (VFP_Error(vc,
-				    "Invalid Gzip data: %s", vg->vz.msg));
+				    "Invalid Gzip data: %s", vgz_msg(vg)));
 			if (dl > 0) {
 				*lp = dl;
 				assert(dp == p);
@@ -610,7 +618,7 @@ vfp_testgunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p,
 				return(VFP_Error(vc, "Junk after gzip data"));
 			if (vr < VGZ_OK)
 				return (VFP_Error(vc,
-				    "Invalid Gzip data: %s", vg->vz.msg));
+				    "Invalid Gzip data: %s", vgz_msg(vg)));
 		} while (!VGZ_IbufEmpty(vg));
 	}
 	VGZ_UpdateObj(vc, vg, VUA_UPDATE);



From phk at FreeBSD.org  Mon May 18 19:12:05 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 18 May 2015 21:12:05 +0200
Subject: [master] a8b61cc fix an error message
Message-ID: <E1YuQSL-0004Aj-LI@project.varnish-software.com>


commit a8b61cc86ab0ee1bf219c84d697b311384f39af9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 18 12:17:31 2015 +0000

    fix an error message

diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 4b353a4..8271368 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -163,8 +163,8 @@ server_start(struct server *s)
 		s->sock = VTCP_listen_on(s->listen, "0", s->depth, &err);
 		if (err != NULL)
 			vtc_log(s->vl, 0,
-			    "Server s listen address cannot be resolved: %s",
-			    err);
+			    "Server listen address (%s) cannot be resolved: %s",
+			    s->listen, err);
 		assert(s->sock > 0);
 		VTCP_myname(s->sock, s->aaddr, sizeof s->aaddr,
 		    s->aport, sizeof s->aport);



From phk at FreeBSD.org  Mon May 18 19:12:05 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 18 May 2015 21:12:05 +0200
Subject: [master] 51f4c23 White-space ocd
Message-ID: <E1YuQSL-0004Ap-MZ@project.varnish-software.com>


commit 51f4c23e049013e5960ee611ff43e9652c9c3929
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 18 17:19:04 2015 +0000

    White-space ocd

diff --git a/bin/varnishtest/tests/l00005.vtc b/bin/varnishtest/tests/l00005.vtc
index 273eaaa..1945934 100644
--- a/bin/varnishtest/tests/l00005.vtc
+++ b/bin/varnishtest/tests/l00005.vtc
@@ -49,9 +49,9 @@ logexpect l1 -v v1 -g session {
 	expect * =	End
 	expect 0 1003	Begin		"^req .* rxreq"
 	expect * =	End
-        expect 0 1002   Begin           "^bereq "
-        expect * =      BereqAcct       "^54 4 58 41 1000 1041$"
-        expect 0 =      End
+	expect 0 1002   Begin		"^bereq "
+	expect * =      BereqAcct       "^54 4 58 41 1000 1041$"
+	expect 0 =      End
 	expect 0 1004	Begin		"^bereq"
 	expect * =	BereqAcct	"^54 4 58 12 0 12$"
 	expect * =	End



From phk at FreeBSD.org  Mon May 18 19:12:05 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 18 May 2015 21:12:05 +0200
Subject: [master] b1dd667 Add a new mode to server instance 'b0',
 where it acts like "a normal webserver" and dispatches a thread for
 each incomming connection.
Message-ID: <E1YuQSL-0004Ay-Nq@project.varnish-software.com>


commit b1dd6679db4f534ba3cd743fb0662965682ab87a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 18 19:07:52 2015 +0000

    Add a new mode to server instance 'b0', where it acts like "a normal
    webserver" and dispatches a thread for each incomming connection.
    
    This is an example use:
    
    	server s0 {
    		loop 10 {
    			rxreq
    			txresp -body "foo1"
    		}
    		rxreq
    		txresp -hdr "Connection: close" -body "foo1"
    		expect_close
    	} -dispatch
    
    Each connection will spawn a dynamically created s%d instance
    starting with s1, s2 ...
    
    Each of these will respond to 11 requests on the accepted connection,
    the last response will have "Connection: close" and they will
    expect the other end (varnish) to do that.
    
    The main limitation on using this for bulk traffic tests is the
    finite and limited size of the vtc_log which varnishtest will
    collect (half a megabyte).
    
    Test b00048 is about as much traffic as will fit in the vtc_log.

diff --git a/bin/varnishtest/tests/b00048.vtc b/bin/varnishtest/tests/b00048.vtc
new file mode 100644
index 0000000..ab9089c
--- /dev/null
+++ b/bin/varnishtest/tests/b00048.vtc
@@ -0,0 +1,49 @@
+varnishtest "Run a lot of transactions through"
+
+server s0 {
+	loop 10 {
+		rxreq
+		txresp -body "foo1"
+	}
+	rxreq
+	txresp -hdr "Connection: close" -body "foo1"
+	expect_close
+} -dispatch
+
+varnish v1 -arg "-p waiter=poll" -vcl+backend {
+	sub vcl_recv {
+		return (pass);
+	}
+	sub vcl_backend_fetch {
+		set bereq.backend = s0;
+	}
+
+} -start
+
+client c1 {
+	loop 20 {
+		txreq -url /c1
+		rxresp
+		expect resp.status == 200
+	}
+} -start
+
+client c2 {
+	loop 20 {
+		txreq -url /c2
+		rxresp
+		expect resp.status == 200
+	}
+} -start
+
+client c3 {
+	loop 20 {
+		txreq -url /c3
+		rxresp
+		expect resp.status == 200
+	}
+} -start
+
+client c1 -wait
+client c2 -wait
+client c3 -wait
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 4b40c90..9053e73 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -643,6 +643,7 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 
 	init_macro();
 	init_sema();
+	init_server();
 
 	/* Apply extmacro definitions */
 	VTAILQ_FOREACH(m, &extmacro_list, list)
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 7837052..e27c056 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -71,6 +71,7 @@ extern int iflg;
 extern unsigned vtc_maxdur;
 
 void init_sema(void);
+void init_server(void);
 
 int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd);
 
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 8271368..73e50ed 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -54,6 +54,7 @@ struct server {
 
 	int			depth;
 	int			sock;
+	int			fd;
 	char			listen[256];
 	char			aaddr[32];
 	char			aport[32];
@@ -61,10 +62,85 @@ struct server {
 	pthread_t		tp;
 };
 
+static pthread_mutex_t		server_mtx;
+
 static VTAILQ_HEAD(, server)	servers =
     VTAILQ_HEAD_INITIALIZER(servers);
 
 /**********************************************************************
+ * Allocate and initialize a server
+ */
+
+static struct server *
+server_new(const char *name)
+{
+	struct server *s;
+
+	AN(name);
+	ALLOC_OBJ(s, SERVER_MAGIC);
+	AN(s);
+	REPLACE(s->name, name);
+	s->vl = vtc_logopen(s->name);
+	AN(s->vl);
+
+	bprintf(s->listen, "%s", "127.0.0.1 0");
+	s->repeat = 1;
+	s->depth = 10;
+	s->sock = -1;
+	s->fd = -1;
+	AZ(pthread_mutex_lock(&server_mtx));
+	VTAILQ_INSERT_TAIL(&servers, s, list);
+	AZ(pthread_mutex_unlock(&server_mtx));
+	return (s);
+}
+
+/**********************************************************************
+ * Clean up a server
+ */
+
+static void
+server_delete(struct server *s)
+{
+
+	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
+	macro_undef(s->vl, s->name, "addr");
+	macro_undef(s->vl, s->name, "port");
+	macro_undef(s->vl, s->name, "sock");
+	vtc_logclose(s->vl);
+	free(s->name);
+	/* XXX: MEMLEAK (?) (VSS ??) */
+	FREE_OBJ(s);
+}
+
+/**********************************************************************
+ * Server listen
+ */
+
+static void
+server_listen(struct server *s)
+{
+	const char *err;
+
+	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
+
+	if (s->sock >= 0)
+		VTCP_close(&s->sock);
+	s->sock = VTCP_listen_on(s->listen, "0", s->depth, &err);
+	if (err != NULL)
+		vtc_log(s->vl, 0,
+		    "Server listen address (%s) cannot be resolved: %s",
+		    s->listen, err);
+	assert(s->sock > 0);
+	VTCP_myname(s->sock, s->aaddr, sizeof s->aaddr,
+	    s->aport, sizeof s->aport);
+	macro_def(s->vl, s->name, "addr", "%s", s->aaddr);
+	macro_def(s->vl, s->name, "port", "%s", s->aport);
+	macro_def(s->vl, s->name, "sock", "%s %s", s->aaddr, s->aport);
+	/* Record the actual port, and reuse it on subsequent starts */
+	bprintf(s->listen, "%s %s", s->aaddr, s->aport);
+}
+
+/**********************************************************************
  * Server thread
  */
 
@@ -83,7 +159,7 @@ server_thread(void *priv)
 
 	vl = vtc_logopen(s->name);
 
-	vtc_log(vl, 2, "Started on %s %s", s->aaddr, s->aport);
+	vtc_log(vl, 2, "Started on %s", s->listen);
 	for (i = 0; i < s->repeat; i++) {
 		if (s->repeat > 1)
 			vtc_log(vl, 3, "Iteration %d", i);
@@ -104,80 +180,92 @@ server_thread(void *priv)
 	return (NULL);
 }
 
+
 /**********************************************************************
- * Allocate and initialize a server
+ * Start the server thread
  */
 
-static struct server *
-server_new(const char *name)
+static void
+server_start(struct server *s)
 {
-	struct server *s;
-
-	AN(name);
-	ALLOC_OBJ(s, SERVER_MAGIC);
-	AN(s);
-	REPLACE(s->name, name);
-	s->vl = vtc_logopen(name);
-	AN(s->vl);
-	if (*s->name != 's')
-		vtc_log(s->vl, 0, "Server name must start with 's'");
-
-	bprintf(s->listen, "%s", "127.0.0.1 0");
-	s->repeat = 1;
-	s->depth = 10;
-	s->sock = -1;
-	VTAILQ_INSERT_TAIL(&servers, s, list);
-	return (s);
+	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
+	vtc_log(s->vl, 2, "Starting server");
+	server_listen(s);
+	vtc_log(s->vl, 1, "Listen on %s", s->listen);
+	s->run = 1;
+	AZ(pthread_create(&s->tp, NULL, server_thread, s));
 }
 
 /**********************************************************************
- * Clean up a server
  */
 
-static void
-server_delete(struct server *s)
+static void *
+server_dispatch_wrk(void *priv)
 {
+	struct server *s;
+	struct vtclog *vl;
+	int j, fd;
 
-	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
-	macro_undef(s->vl, s->name, "addr");
-	macro_undef(s->vl, s->name, "port");
-	macro_undef(s->vl, s->name, "sock");
-	vtc_logclose(s->vl);
-	free(s->name);
-	/* XXX: MEMLEAK (?) (VSS ??) */
-	FREE_OBJ(s);
+	CAST_OBJ_NOTNULL(s, priv, SERVER_MAGIC);
+	assert(s->sock < 0);
+
+	vl = vtc_logopen(s->name);
+
+	fd = s->fd;
+
+	vtc_log(vl, 3, "start with fd %d", fd);
+	fd = http_process(vl, s->spec, fd, &s->sock);
+	vtc_log(vl, 3, "shutting fd %d", fd);
+	j = shutdown(fd, SHUT_WR);
+	if (!VTCP_Check(j))
+		vtc_log(vl, 0, "Shutdown failed: %s", strerror(errno));
+	VTCP_close(&s->fd);
+	vtc_log(vl, 2, "Ending");
+	return (NULL);
 }
 
-/**********************************************************************
- * Start the server thread
- */
+static void *
+server_dispatch_thread(void *priv)
+{
+	struct server *s, *s2;
+	int sn = 1, fd;
+	char snbuf[8];
+	struct vtclog *vl;
+	struct sockaddr_storage addr_s;
+	struct sockaddr *addr;
+	socklen_t l;
+
+	CAST_OBJ_NOTNULL(s, priv, SERVER_MAGIC);
+	assert(s->sock >= 0);
+
+	vl = vtc_logopen(s->name);
+	vtc_log(vl, 2, "Dispatch started on %s", s->listen);
+
+	while (1) {
+		addr = (void*)&addr_s;
+		l = sizeof addr_s;
+		fd = accept(s->sock, addr, &l);
+		if (fd < 0)
+			vtc_log(vl, 0, "Accepted failed: %s", strerror(errno));
+		bprintf(snbuf, "s%d", sn++);
+		vtc_log(vl, 3, "dispatch fd %d -> %s", fd, snbuf);
+		s2 = server_new(snbuf);
+		s2->spec = s->spec;
+		strcpy(s2->listen, s->listen);
+		s2->fd = fd;
+		s2->run = 1;
+		AZ(pthread_create(&s2->tp, NULL, server_dispatch_wrk, s2));
+	}
+}
 
 static void
-server_start(struct server *s)
+server_dispatch(struct server *s)
 {
-	const char *err;
-
 	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
-	vtc_log(s->vl, 2, "Starting server");
-	if (s->sock < 0) {
-		s->sock = VTCP_listen_on(s->listen, "0", s->depth, &err);
-		if (err != NULL)
-			vtc_log(s->vl, 0,
-			    "Server listen address (%s) cannot be resolved: %s",
-			    s->listen, err);
-		assert(s->sock > 0);
-		VTCP_myname(s->sock, s->aaddr, sizeof s->aaddr,
-		    s->aport, sizeof s->aport);
-		macro_def(s->vl, s->name, "addr", "%s", s->aaddr);
-		macro_def(s->vl, s->name, "port", "%s", s->aport);
-		macro_def(s->vl, s->name, "sock", "%s %s", s->aaddr, s->aport);
-
-		/* Record the actual port, and reuse it on subsequent starts */
-		bprintf(s->listen, "%s %s", s->aaddr, s->aport);
-	}
-	vtc_log(s->vl, 1, "Listen on %s", s->listen);
+	server_listen(s);
+	vtc_log(s->vl, 2, "Starting dispatch server");
 	s->run = 1;
-	AZ(pthread_create(&s->tp, NULL, server_thread, s));
+	AZ(pthread_create(&s->tp, NULL, server_dispatch_thread, s));
 }
 
 /**********************************************************************
@@ -207,7 +295,7 @@ server_wait(struct server *s)
 	void *res;
 
 	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
-	vtc_log(s->vl, 2, "Waiting for server");
+	vtc_log(s->vl, 2, "Waiting for server (%d/%d)", s->sock, s->fd);
 	AZ(pthread_join(s->tp, &res));
 	if (res != NULL && !vtc_stop)
 		vtc_log(s->vl, 0, "Server returned \"%p\"",
@@ -225,11 +313,13 @@ cmd_server_genvcl(struct vsb *vsb)
 {
 	struct server *s;
 
+	AZ(pthread_mutex_lock(&server_mtx));
 	VTAILQ_FOREACH(s, &servers, list) {
 		VSB_printf(vsb,
 		    "backend %s { .host = \"%s\"; .port = \"%s\"; }\n",
 		    s->name, s->aaddr, s->aport);
 	}
+	AZ(pthread_mutex_unlock(&server_mtx));
 }
 
 
@@ -240,7 +330,7 @@ cmd_server_genvcl(struct vsb *vsb)
 void
 cmd_server(CMD_ARGS)
 {
-	struct server *s, *s2;
+	struct server *s;
 
 	(void)priv;
 	(void)cmd;
@@ -248,9 +338,15 @@ cmd_server(CMD_ARGS)
 
 	if (av == NULL) {
 		/* Reset and free */
-		VTAILQ_FOREACH_SAFE(s, &servers, list, s2) {
-			CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
-			VTAILQ_REMOVE(&servers, s, list);
+		while (1) {
+			AZ(pthread_mutex_lock(&server_mtx));
+			s = VTAILQ_FIRST(&servers);
+			CHECK_OBJ_ORNULL(s, SERVER_MAGIC);
+			if (s != NULL)
+				VTAILQ_REMOVE(&servers, s, list);
+			AZ(pthread_mutex_unlock(&server_mtx));
+			if (s == NULL)
+				break;
 			if (s->run) {
 				(void)pthread_cancel(s->tp);
 				server_wait(s);
@@ -265,9 +361,17 @@ cmd_server(CMD_ARGS)
 	AZ(strcmp(av[0], "server"));
 	av++;
 
+	if (*av[0] != 's') {
+		fprintf(stderr, "Server name must start with 's' (is: %s)\n",
+		    av[0]);
+		exit(1);
+	}
+
+	AZ(pthread_mutex_lock(&server_mtx));
 	VTAILQ_FOREACH(s, &servers, list)
 		if (!strcmp(s->name, av[0]))
 			break;
+	AZ(pthread_mutex_unlock(&server_mtx));
 	if (s == NULL)
 		s = server_new(av[0]);
 	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
@@ -312,8 +416,23 @@ cmd_server(CMD_ARGS)
 			server_start(s);
 			continue;
 		}
+		if (!strcmp(*av, "-dispatch")) {
+			if (strcmp(s->name, "s0")) {
+				fprintf(stderr,
+				    "server -parallel only works on s0\n");
+				exit(1);
+			}
+			server_dispatch(s);
+			continue;
+		}
 		if (**av == '-')
 			vtc_log(s->vl, 0, "Unknown server argument: %s", *av);
 		s->spec = *av;
 	}
 }
+
+void
+init_server(void)
+{
+	AZ(pthread_mutex_init(&server_mtx, NULL));
+}



From phk at FreeBSD.org  Mon May 18 19:37:04 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 18 May 2015 21:37:04 +0200
Subject: [master] 51c0b18 Don't use libgzip internal #definery outside libgzip
Message-ID: <E1YuQqW-0006co-CF@project.varnish-software.com>


commit 51c0b18fef86b50cf81902fa4d9f820695c833e9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 18 19:36:39 2015 +0000

    Don't use libgzip internal #definery outside libgzip

diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 156b66f..8079e2b 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -64,7 +64,7 @@ struct vgz {
 	z_stream		vz;
 };
 
-static inline z_const char *
+static const char *
 vgz_msg(const struct vgz *vg)
 {
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index d3cbe4e..d3ab87c 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -141,6 +141,7 @@ vwp_main(void *priv)
 			vwp->hpoll--;
 		v = poll(vwp->pollfd, vwp->hpoll + 1, -1);
 		assert(v >= 0);
+usleep(1500000);
 		v2 = v;
 		now = VTIM_real();
 		idle = now - *vwp->waiter->tmo;



From phk at FreeBSD.org  Mon May 18 19:38:09 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 18 May 2015 21:38:09 +0200
Subject: [master] ffc28ea A random attempt at debugging that shouldn't have
 been committed
Message-ID: <E1YuQrZ-0006kj-DT@project.varnish-software.com>


commit ffc28ea95f81faf9cd07f53ff700cf91e7c8e92e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 18 19:37:49 2015 +0000

    A random attempt at debugging that shouldn't have been committed

diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index d3ab87c..d3cbe4e 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -141,7 +141,6 @@ vwp_main(void *priv)
 			vwp->hpoll--;
 		v = poll(vwp->pollfd, vwp->hpoll + 1, -1);
 		assert(v >= 0);
-usleep(1500000);
 		v2 = v;
 		now = VTIM_real();
 		idle = now - *vwp->waiter->tmo;



From phk at FreeBSD.org  Mon May 18 19:53:34 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 18 May 2015 21:53:34 +0200
Subject: [master] ba2d2e0 Add a needless return for GCC's pointless warning.
Message-ID: <E1YuR6U-0008CY-4F@project.varnish-software.com>


commit ba2d2e001bfad35cd1abf71201e7a3a2bd1aea3c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 18 19:53:14 2015 +0000

    Add a needless return for GCC's pointless warning.

diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 73e50ed..9700546 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -256,6 +256,7 @@ server_dispatch_thread(void *priv)
 		s2->run = 1;
 		AZ(pthread_create(&s2->tp, NULL, server_dispatch_wrk, s2));
 	}
+	return(NULL);
 }
 
 static void



From phk at FreeBSD.org  Mon May 18 20:48:18 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 18 May 2015 22:48:18 +0200
Subject: [master] 490a552 We cannot expect there to be a full second between
 the Bereq/Beresp timestamps,
 when the Bereq timestamp is taken after our write(2) to the socket,
 which may have caused us to loose the CPU.
Message-ID: <E1YuRxS-0005Ct-O3@project.varnish-software.com>


commit 490a55223c7d574df38622ea62d77b072293733d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 18 20:36:29 2015 +0000

    We cannot expect there to be a full second between the Bereq/Beresp
    timestamps, when the Bereq timestamp is taken after our write(2)
    to the socket, which may have caused us to loose the CPU.

diff --git a/bin/varnishtest/tests/s00004.vtc b/bin/varnishtest/tests/s00004.vtc
index 169d6ab..cfdc766 100644
--- a/bin/varnishtest/tests/s00004.vtc
+++ b/bin/varnishtest/tests/s00004.vtc
@@ -36,7 +36,7 @@ logexpect l1 -v v1 -g request {
 	expect 0 1002	Begin		bereq
 	expect * =	Timestamp	{Start: \S+ 0\.000000 0\.000000}
 	expect * =	Timestamp	{Bereq: \S+ 0\.\d+ 0\.\d+}
-	expect * =	Timestamp	{Beresp: \S+ 1\.\d+ 1\.\d+}
+	expect * =	Timestamp	{Beresp: \S+ 1\.\d+ [01]\.\d+}
 	expect * =	Timestamp	{BerespBody: \S+ 2\.\d+ 1\.\d+}
 	expect * =	End
 	expect 0 1003	Begin		req 1001 restart



From lkarsten at varnish-software.com  Wed May 20 11:21:11 2015
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Wed, 20 May 2015 13:21:11 +0200
Subject: [master] 85729c5 Squash trailing whitespaces.
Message-ID: <E1Yv23j-00081c-N3@project.varnish-software.com>


commit 85729c51cfc184e94c8239d7e5bdc4c92feccc0d
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed May 20 13:20:28 2015 +0200

    Squash trailing whitespaces.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index ff50441..2f5e5dd 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -45,7 +45,7 @@ Requires(postun): systemd-units
 BuildRequires: systemd-units
 %endif
 
-# Varnish actually needs gcc installed to work. It uses the C compiler 
+# Varnish actually needs gcc installed to work. It uses the C compiler
 # at runtime to compile the VCL configuration files. This is by design.
 Requires: gcc
 
@@ -272,13 +272,13 @@ exit 0
 %else
 /sbin/chkconfig --add varnish
 /sbin/chkconfig --add varnishlog
-/sbin/chkconfig --add varnishncsa 
+/sbin/chkconfig --add varnishncsa
 %endif
 test -f /etc/varnish/secret || (uuidgen > /etc/varnish/secret && chmod 0600 /etc/varnish/secret)
 
 %triggerun -- varnish < 3.0.2-1
 # Save the current service runlevel info
-# User must manually run systemd-sysv-convert --apply varnish 
+# User must manually run systemd-sysv-convert --apply varnish
 # to migrate them to systemd targets
 %{_bindir}/systemd-sysv-convert --save varnish >/dev/null 2>&1 ||:
 
@@ -301,7 +301,7 @@ if [ $1 -lt 1 ]; then
   /sbin/service varnishncsa stop > /dev/null 2>%1
   /sbin/chkconfig --del varnish
   /sbin/chkconfig --del varnishlog
-  /sbin/chkconfig --del varnishncsa 
+  /sbin/chkconfig --del varnishncsa
   %endif
 fi
 
@@ -317,10 +317,10 @@ fi
 * Tue Mar 06 2012 Ingvar Hagelund <ingvar at redpill-linpro.com> - 3.0.2-1
 - New upstream version 3.0.2
 - Removed INSTALL as requested by rpmlint
-- Added a ld.so.conf.d fragment file listing libdir/varnish 
+- Added a ld.so.conf.d fragment file listing libdir/varnish
 - Removed redundant doc/html/_sources
 - systemd support from fedora 17
-- Stopped using macros for make and install, according to 
+- Stopped using macros for make and install, according to
   Fedora's packaging guidelines
 - Changes merged from upstream:
   - Added suse_version macro
@@ -408,8 +408,8 @@ fi
 
 * Tue Apr 06 2010 Ingvar Hagelund <ingvar at linpro.no> - 2.1.0-1
 - New upstream release; note: Configuration changes, see the README
-- Removed unneeded patches 
-- CVE-2009-2936: Added a patch from Debian that adds the -S option 
+- Removed unneeded patches
+- CVE-2009-2936: Added a patch from Debian that adds the -S option
   to the varnisdh(1) manpage and to the sysconfig defaults, thus
   password-protecting the admin interface port (#579536,#579533)
 - Generates that password in the post script, requires mkpasswd
@@ -440,7 +440,7 @@ fi
 - Added a s390 specific patch to libjemalloc.
 
 * Fri Mar 27 2009 Ingvar Hagelund <ingvar at linpro.no> - 2.0.4-1
-  New upstream release 2.0.4 
+  New upstream release 2.0.4
 
 * Wed Feb 25 2009 Fedora Release Engineering <rel-eng at lists.fedoraproject.org> - 2.0.3-2
 - Rebuilt for https://fedoraproject.org/wiki/Fedora_11_Mass_Rebuild
@@ -539,7 +539,7 @@ fi
 
 * Fri Sep 07 2007 Ingvar Hagelund <ingvar at linpro.no> - 1.1.1-3
 - Added a patch, changeset 1913 from svn trunk. This makes varnish
-  more stable under specific loads. 
+  more stable under specific loads.
 
 * Thu Sep 06 2007 Ingvar Hagelund <ingvar at linpro.no> - 1.1.1-2
 - Removed autogen call (only diff from relase tarball)
@@ -584,7 +584,7 @@ fi
 - Fixed inconsistently use of brackets in macros
 - Added a condrestart to the initscript
 - All manfiles included, not just the compressed ones
-- Removed explicit requirement for ncurses. rpmbuild figures out the 
+- Removed explicit requirement for ncurses. rpmbuild figures out the
   correct deps by itself.
 - Added ulimit value to initskript and sysconfig file
 - Many thanks to Matthias Saou for valuable input



From nils.goroll at uplex.de  Thu May 21 10:49:42 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 21 May 2015 12:49:42 +0200
Subject: [master] 730e185 clarifying comments
Message-ID: <E1YvO2o-0008QH-Km@project.varnish-software.com>


commit 730e185aa1d808a9c3f9fccece6747158ba84d13
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed May 20 12:18:00 2015 +0200

    clarifying comments

diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index d3ea762..df58054 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -113,7 +113,13 @@ RFC2616_Ttl(struct busyobj *bo, double now)
 		break;
 	case 302: /* Moved Temporarily */
 	case 307: /* Temporary Redirect */
-		expp->ttl = -1.;
+		/*
+		 * https://tools.ietf.org/html/rfc7231#section-6.1
+		 *
+		 * Do not apply the default ttl, only set a ttl if Cache-Control
+		 * or Expires are present. Uncacheable otherwise.
+		 */
+		expp->ttl = -1.;	/* fall through */
 	case 200: /* OK */
 	case 203: /* Non-Authoritative Information */
 	case 204: /* No Content */



From phk at FreeBSD.org  Thu May 21 16:48:51 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 18:48:51 +0200
Subject: [master] 54fbace Split waiter functions into the ones which deal with
 waiters (Waiter_*) and the ones which deal with fd's being waited on (Wait_*)
Message-ID: <E1YvTeN-000564-BX@project.varnish-software.com>


commit 54fbace30c3aa15816c418dd836aa8ef350760f4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 19 09:19:06 2015 +0000

    Split waiter functions into the ones which deal with waiters (Waiter_*)
    and the ones which deal with fd's being waited on (Wait_*)

diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index 518870e..ef037c9 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -178,7 +178,7 @@ VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6)
 	VTAILQ_INIT(&tp->killlist);
 	VTAILQ_INSERT_HEAD(&pools, tp, list);
 	tp->timeout = 60;
-	tp->waiter = Wait_New(tcp_handle, &tp->timeout);
+	tp->waiter = Waiter_New(tcp_handle, &tp->timeout);
 	return (tp);
 }
 
@@ -228,7 +228,7 @@ VBT_Rel(struct tcp_pool **tpp)
 	Lck_Delete(&tp->mtx);
 	AZ(tp->n_conn);
 	AZ(tp->n_kill);
-	Wait_Destroy(&tp->waiter);
+	Waiter_Destroy(&tp->waiter);
 
 	FREE_OBJ(tp);
 }
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index bfe99e3..7baee82 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -213,7 +213,7 @@ child_main(void)
 	PAN_Init();
 	VFP_Init();
 
-	Wait_Init();
+	Waiter_Init();
 
 	VCL_Init();
 
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 35e738b..ee1091c 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -550,7 +550,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 
 	VSB_printf(pan_vsp, "version = %s\n", VCS_version);
 	VSB_printf(pan_vsp, "ident = %s,%s\n",
-	    VSB_data(vident) + 1, Wait_GetName());
+	    VSB_data(vident) + 1, Waiter_GetName());
 
 	pan_backtrace();
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 6153f84..37a6109 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -589,7 +589,7 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	bprintf(nb, "sess%u", pool_no);
 	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool,
 	    &cache_param->workspace_session);
-	pp->http1_waiter = Wait_New(ses_handle, &cache_param->timeout_idle);
+	pp->http1_waiter = Waiter_New(ses_handle, &cache_param->timeout_idle);
 
 	VCA_New_SessPool(wp, pp);
 	return (pp);
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index b005eb0..c65adcd 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -77,7 +77,7 @@ wait_poker_thread(void *arg)
 }
 
 const char *
-Wait_GetName(void)
+Waiter_GetName(void)
 {
 
 	if (waiter != NULL)
@@ -87,7 +87,7 @@ Wait_GetName(void)
 }
 
 struct waiter *
-Wait_New(waiter_handle_f *func, volatile double *tmo)
+Waiter_New(waiter_handle_f *func, volatile double *tmo)
 {
 	struct waiter *w;
 
@@ -120,7 +120,7 @@ Wait_New(waiter_handle_f *func, volatile double *tmo)
 }
 
 void
-Wait_Destroy(struct waiter **wp)
+Waiter_Destroy(struct waiter **wp)
 {
 	struct waiter *w;
 	struct waited *wx = NULL;
@@ -292,7 +292,7 @@ Wait_Handle(struct waiter *w, struct waited *wp, enum wait_event ev, double now)
 }
 
 void
-Wait_Init(void)
+Waiter_Init(void)
 {
 
 	Lck_New(&wait_mtx, lck_misc);
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
index 953df8e..db69e26 100644
--- a/bin/varnishd/waiter/waiter.h
+++ b/bin/varnishd/waiter/waiter.h
@@ -58,10 +58,10 @@ typedef void waiter_handle_f(struct waited *, enum wait_event, double now);
 
 /* cache_waiter.c */
 int Wait_Enter(const struct waiter *, struct waited *);
-struct waiter *Wait_New(waiter_handle_f *, volatile double *timeout);
-void Wait_Destroy(struct waiter **);
-const char *Wait_GetName(void);
-void Wait_Init(void);
+struct waiter *Waiter_New(waiter_handle_f *, volatile double *timeout);
+void Waiter_Destroy(struct waiter **);
+const char *Waiter_GetName(void);
+void Waiter_Init(void);
 
 /* mgt_waiter.c */
 int Wait_Argument(struct vsb *vsb, const char *arg);



From phk at FreeBSD.org  Thu May 21 16:48:51 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 18:48:51 +0200
Subject: [master] c960cd1 sort functions into Wait_ and Waited_
Message-ID: <E1YvTeN-00056B-Cp@project.varnish-software.com>


commit c960cd16387c21ca4fe4745074a137d1eb6650e6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 19 09:31:19 2015 +0000

    sort functions into Wait_ and Waited_

diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index c65adcd..c1a03fa 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -76,91 +76,6 @@ wait_poker_thread(void *arg)
 	NEEDLESS_RETURN(NULL);
 }
 
-const char *
-Waiter_GetName(void)
-{
-
-	if (waiter != NULL)
-		return (waiter->name);
-	else
-		return ("no_waiter");
-}
-
-struct waiter *
-Waiter_New(waiter_handle_f *func, volatile double *tmo)
-{
-	struct waiter *w;
-
-	AN(waiter);
-	AN(waiter->name);
-	AN(waiter->init);
-
-	w = calloc(1, sizeof (struct waiter) + waiter->size);
-	AN(w);
-	INIT_OBJ(w, WAITER_MAGIC);
-	w->priv = (void*)(w + 1);
-	w->impl = waiter;
-	w->func = func;
-	w->tmo = tmo;
-	w->pipes[0] = w->pipes[1] = -1;
-	VTAILQ_INIT(&w->waithead);
-
-	waiter->init(w);
-	AN(w->impl->pass || w->pipes[1] >= 0);
-
-	Lck_Lock(&wait_mtx);
-	VTAILQ_INSERT_TAIL(&waiters, w, list);
-	nwaiters++;
-
-	/* We assume all waiters either use pipes or don't use pipes */
-	if (w->pipes[1] >= 0 && nwaiters == 1)
-		AZ(pthread_create(&wait_thr, NULL, wait_poker_thread, NULL));
-	Lck_Unlock(&wait_mtx);
-	return (w);
-}
-
-void
-Waiter_Destroy(struct waiter **wp)
-{
-	struct waiter *w;
-	struct waited *wx = NULL;
-	int written;
-	double now;
-
-	AN(wp);
-	w = *wp;
-	*wp = NULL;
-	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
-
-	Lck_Lock(&wait_mtx);
-	VTAILQ_REMOVE(&waiters, w, list);
-	w->dismantle = 1;
-	Lck_Unlock(&wait_mtx);
-
-	if (w->pipes[1] >= 0) {
-		while (1) {
-			written = write(w->pipes[1], &wx, sizeof wx);
-			if (written == sizeof wx)
-				break;
-			(void)usleep(10000);
-		}
-	}
-	AN(w->impl->fini);
-	w->impl->fini(w);
-	now = VTIM_real();
-	while (1) {
-		wx = VTAILQ_FIRST(&w->waithead);
-		if (wx == NULL)
-			break;
-		VTAILQ_REMOVE(&w->waithead, wx, list);
-		if (wx == w->pipe_w)
-			FREE_OBJ(wx);
-		else
-			w->func(wx, WAITER_CLOSE, now);
-	}
-	FREE_OBJ(w);
-}
-
 void
 Wait_UsePipe(struct waiter *w)
 {
@@ -291,6 +206,93 @@ Wait_Handle(struct waiter *w, struct waited *wp, enum wait_event ev, double now)
 	wait_updidle(w, now);
 }
 
+/**********************************************************************/
+
+const char *
+Waiter_GetName(void)
+{
+
+	if (waiter != NULL)
+		return (waiter->name);
+	else
+		return ("no_waiter");
+}
+
+struct waiter *
+Waiter_New(waiter_handle_f *func, volatile double *tmo)
+{
+	struct waiter *w;
+
+	AN(waiter);
+	AN(waiter->name);
+	AN(waiter->init);
+
+	w = calloc(1, sizeof (struct waiter) + waiter->size);
+	AN(w);
+	INIT_OBJ(w, WAITER_MAGIC);
+	w->priv = (void*)(w + 1);
+	w->impl = waiter;
+	w->func = func;
+	w->tmo = tmo;
+	w->pipes[0] = w->pipes[1] = -1;
+	VTAILQ_INIT(&w->waithead);
+
+	waiter->init(w);
+	AN(w->impl->pass || w->pipes[1] >= 0);
+
+	Lck_Lock(&wait_mtx);
+	VTAILQ_INSERT_TAIL(&waiters, w, list);
+	nwaiters++;
+
+	/* We assume all waiters either use pipes or don't use pipes */
+	if (w->pipes[1] >= 0 && nwaiters == 1)
+		AZ(pthread_create(&wait_thr, NULL, wait_poker_thread, NULL));
+	Lck_Unlock(&wait_mtx);
+	return (w);
+}
+
+void
+Waiter_Destroy(struct waiter **wp)
+{
+	struct waiter *w;
+	struct waited *wx = NULL;
+	int written;
+	double now;
+
+	AN(wp);
+	w = *wp;
+	*wp = NULL;
+	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
+
+	Lck_Lock(&wait_mtx);
+	VTAILQ_REMOVE(&waiters, w, list);
+	w->dismantle = 1;
+	Lck_Unlock(&wait_mtx);
+
+	if (w->pipes[1] >= 0) {
+		while (1) {
+			written = write(w->pipes[1], &wx, sizeof wx);
+			if (written == sizeof wx)
+				break;
+			(void)usleep(10000);
+		}
+	}
+	AN(w->impl->fini);
+	w->impl->fini(w);
+	now = VTIM_real();
+	while (1) {
+		wx = VTAILQ_FIRST(&w->waithead);
+		if (wx == NULL)
+			break;
+		VTAILQ_REMOVE(&w->waithead, wx, list);
+		if (wx == w->pipe_w)
+			FREE_OBJ(wx);
+		else
+			w->func(wx, WAITER_CLOSE, now);
+	}
+	FREE_OBJ(w);
+}
+
 void
 Waiter_Init(void)
 {



From phk at FreeBSD.org  Thu May 21 16:48:51 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 18:48:51 +0200
Subject: [master] 77dd6bc Disable all waiters but the 'poll' waiter.
Message-ID: <E1YvTeN-00056L-Du@project.varnish-software.com>


commit 77dd6bc31d4da5db226b09378d97bc9a2d07301a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 19 09:35:04 2015 +0000

    Disable all waiters but the 'poll' waiter.
    
    Don't worry, they'll will come back later.

diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index f1e8bd9..2e85b87 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -31,6 +31,7 @@
  * write the session pointer to a pipe which the event engine monitors.
  */
 
+#if 0
 #include "config.h"
 
 #if defined(HAVE_EPOLL_CTL)
@@ -165,3 +166,4 @@ const struct waiter_impl waiter_epoll = {
 };
 
 #endif /* defined(HAVE_EPOLL_CTL) */
+#endif
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index 8985c1a..c28b2de 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -31,6 +31,7 @@
  * write the session pointer to a pipe which the event engine monitors.
  */
 
+#if 0
 #include "config.h"
 
 #if defined(HAVE_KQUEUE)
@@ -214,3 +215,4 @@ const struct waiter_impl waiter_kqueue = {
 };
 
 #endif /* defined(HAVE_KQUEUE) */
+#endif
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index 64aa094..a41c8fa 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -29,6 +29,7 @@
  *
  */
 
+#if 0
 #include "config.h"
 
 #if defined(HAVE_PORT_CREATE)
@@ -291,3 +292,4 @@ const struct waiter_impl waiter_ports = {
 };
 
 #endif /* defined(HAVE_PORT_CREATE) */
+#endif
diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c
index 18e370c..a97425a 100644
--- a/bin/varnishd/waiter/mgt_waiter.c
+++ b/bin/varnishd/waiter/mgt_waiter.c
@@ -39,6 +39,7 @@
 #include "waiter/waiter_priv.h"
 
 static const struct waiter_impl *const waiter_impls[] = {
+#if 0
     #if defined(HAVE_KQUEUE)
 	&waiter_kqueue,
     #endif
@@ -48,6 +49,7 @@ static const struct waiter_impl *const waiter_impls[] = {
     #if defined(HAVE_PORT_CREATE)
 	&waiter_ports,
     #endif
+#endif
 	&waiter_poll,
 	NULL,
 };



From phk at FreeBSD.org  Thu May 21 16:48:51 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 18:48:51 +0200
Subject: [master] 5947441 Add a FALL-THROUGH comment
Message-ID: <E1YvTeN-00056U-Es@project.varnish-software.com>


commit 59474412594ed9257abac9dbab6c10f929dc9a45
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 19 10:06:50 2015 +0000

    Add a FALL-THROUGH comment

diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index df58054..22f1a90 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -120,6 +120,7 @@ RFC2616_Ttl(struct busyobj *bo, double now)
 		 * or Expires are present. Uncacheable otherwise.
 		 */
 		expp->ttl = -1.;	/* fall through */
+		/* FALL-THROUGH */
 	case 200: /* OK */
 	case 203: /* Non-Authoritative Information */
 	case 204: /* No Content */



From phk at FreeBSD.org  Thu May 21 16:48:51 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 18:48:51 +0200
Subject: [master] aeee5b5 If we rely on VSL order,
 space the requests a fraction of a second to ensure this order
Message-ID: <E1YvTeN-00056d-Fs@project.varnish-software.com>


commit aeee5b5ae13f6801ad1e41819faaa45fb92dc04d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 07:29:56 2015 +0000

    If we rely on VSL order, space the requests a fraction of a second
    to ensure this order

diff --git a/bin/varnishtest/tests/c00066.vtc b/bin/varnishtest/tests/c00066.vtc
index 18a9b67..ec88da6 100644
--- a/bin/varnishtest/tests/c00066.vtc
+++ b/bin/varnishtest/tests/c00066.vtc
@@ -44,11 +44,13 @@ client c1 {
 	rxresp
 	expect resp.status == 200
 	expect resp.msg == "OK"
+	delay .1
 
 	txreq -url "/synth"
 	rxresp
 	expect resp.status == 200
 	expect resp.msg == "Synth test"
+	delay .1
 
 	txreq -url "/error"
 	rxresp



From phk at FreeBSD.org  Thu May 21 16:48:51 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 18:48:51 +0200
Subject: [master] cbc829f Scrape back the waiter API,
 and rewrite the poll waiter to handle sparse fd-sets efficiently.
Message-ID: <E1YvTeN-00056m-H2@project.varnish-software.com>


commit cbc829fef3f9fa65ce1c945ce1d18f507adf8611
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 09:22:17 2015 +0000

    Scrape back the waiter API, and rewrite the poll waiter to handle
    sparse fd-sets efficiently.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index cad336e..89d4cd9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -374,7 +374,6 @@ struct waited {
 	unsigned		magic;
 #define WAITED_MAGIC		0x1743992d
 	int			fd;
-	VTAILQ_ENTRY(waited)	list;
 	void			*ptr;
 	double			idle;
 };
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 7baee82..00db49d 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -37,7 +37,6 @@
 
 #include "vcli_priv.h"
 #include "vrnd.h"
-#include "waiter/waiter.h"
 
 #include "hash/hash_slinger.h"
 
@@ -213,8 +212,6 @@ child_main(void)
 	PAN_Init();
 	VFP_Init();
 
-	Waiter_Init();
-
 	VCL_Init();
 
 	HTTP_Init();
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index c1a03fa..3373d0c 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -36,174 +36,17 @@
 
 #include "cache/cache.h"
 
-#include "vfil.h"
-#include "vtim.h"
-
 #include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
 
-#define NEV 8192
-
-static VTAILQ_HEAD(, waiter)	waiters = VTAILQ_HEAD_INITIALIZER(waiters);
-static int			nwaiters;
-static struct lock		wait_mtx;
-static pthread_t		wait_thr;
-
-static void *
-wait_poker_thread(void *arg)
-{
-	struct waiter *w;
-	double now;
-
-	(void)arg;
-	THR_SetName("Waiter timer");
-	while (1) {
-		/* Avoid thundering herds and resonances */
-		(void)usleep(990013/nwaiters);
-
-		now = VTIM_real();
-
-		Lck_Lock(&wait_mtx);
-		w = VTAILQ_FIRST(&waiters);
-		VTAILQ_REMOVE(&waiters, w, list);
-		VTAILQ_INSERT_TAIL(&waiters, w, list);
-		assert(w->pipes[1] >= 0);
-
-		if (w->next_idle + *w->tmo < now)
-			(void)write(w->pipes[1], &w->pipe_w, sizeof w->pipe_w);
-		Lck_Unlock(&wait_mtx);
-	}
-	NEEDLESS_RETURN(NULL);
-}
-
-void
-Wait_UsePipe(struct waiter *w)
-{
-	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
-
-	AN(waiter->inject);
-	AZ(pipe(w->pipes));
-	AZ(VFIL_nonblocking(w->pipes[0]));
-	AZ(VFIL_nonblocking(w->pipes[1]));
-	ALLOC_OBJ(w->pipe_w, WAITED_MAGIC);
-	w->pipe_w->fd = w->pipes[0];
-	w->pipe_w->idle = 0;
-	VTAILQ_INSERT_HEAD(&w->waithead, w->pipe_w, list);
-	waiter->inject(w, w->pipe_w);
-}
-
 int
 Wait_Enter(const struct waiter *w, struct waited *wp)
 {
-	ssize_t written;
-	uintptr_t up;
 
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
-	assert(wp->fd > 0);		// stdin never comes here
-	AZ(w->dismantle);
-
-	if (w->impl->pass != NULL)
-		return (w->impl->pass(w->priv, wp));
-
-	assert(w->pipes[1] > 0);
-
-	up = (uintptr_t)wp;
-	written = write(w->pipes[1], &up, sizeof up);
-	if (written != sizeof up && (errno == EAGAIN || errno == EWOULDBLOCK))
-		return (-1);
-	assert (written == sizeof up);
-	return (0);
-}
-
-static void
-wait_updidle(struct waiter *w, double now)
-{
-	struct waited *wp;
-
-	wp = VTAILQ_FIRST(&w->waithead);
-	if (wp == NULL)
-		return;
-	if (wp == w->pipe_w) {
-		VTAILQ_REMOVE(&w->waithead, wp, list);
-		VTAILQ_INSERT_TAIL(&w->waithead, wp, list);
-		wp->idle = now;
-		wp = VTAILQ_FIRST(&w->waithead);
-	}
-	w->next_idle = wp->idle;
-}
-
-void
-Wait_Handle(struct waiter *w, struct waited *wp, enum wait_event ev, double now)
-{
-	uintptr_t ss[NEV];
-	struct waited *wp2;
-	int i, j, dotimer = 0;
-
-	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
-	CHECK_OBJ_ORNULL(wp, WAITED_MAGIC);
-
-	if (wp != NULL) {
-		if (wp == w->pipe_w) {
-			w->do_pipe = 1;
-			VTAILQ_REMOVE(&w->waithead, w->pipe_w, list);
-			wp->idle = now;
-			VTAILQ_INSERT_TAIL(&w->waithead, w->pipe_w, list);
-		} else {
-			if (w->impl->evict != NULL)
-				w->impl->evict(w, wp);
-
-			VTAILQ_REMOVE(&w->waithead, wp, list);
-			w->func(wp, ev, now);
-			wait_updidle(w, now);
-		}
-		return;
-	}
-
-	AZ(wp);
-
-	if (!w->do_pipe)
-		return;
-
-	w->do_pipe = 0;
-
-	i = read(w->pipes[0], ss, sizeof ss);
-	if (i == -1 && errno == EAGAIN)
-		return;
-
-	for (j = 0; i >= sizeof ss[0]; j++, i -= sizeof ss[0]) {
-		if (ss[j] == 0) {
-			AN(w->dismantle);
-			continue;
-		}
-		ss[j] &= ~1;
-		CAST_OBJ_NOTNULL(wp2, (void*)ss[j], WAITED_MAGIC);
-		if (wp2 == w->pipe_w) {
-			dotimer = 1;
-		} else {
-			assert(wp2->fd >= 0);
-			VTAILQ_INSERT_TAIL(&w->waithead, wp2, list);
-			w->impl->inject(w, wp2);
-		}
-	}
-	AZ(i);
-
-	wait_updidle(w, now);
-
-	if (!dotimer)
-		return;
-
-	VTAILQ_FOREACH_SAFE(wp, &w->waithead, list, wp2) {
-		if (wp == w->pipe_w)
-			continue;
-		if (wp->idle + *w->tmo > now)
-			break;
-		if (w->impl->evict != NULL)
-			w->impl->evict(w, wp);
-		VTAILQ_REMOVE(&w->waithead, wp, list);
-		w->func(wp, WAITER_TIMEOUT, now);
-	}
-	wait_updidle(w, now);
+	assert(wp->fd > 0);			// stdin never comes here
+	return (w->impl->enter(w->priv, wp));
 }
 
 /**********************************************************************/
@@ -215,7 +58,7 @@ Waiter_GetName(void)
 	if (waiter != NULL)
 		return (waiter->name);
 	else
-		return ("no_waiter");
+		return ("(No Waiter?)");
 }
 
 struct waiter *
@@ -226,6 +69,8 @@ Waiter_New(waiter_handle_f *func, volatile double *tmo)
 	AN(waiter);
 	AN(waiter->name);
 	AN(waiter->init);
+	AN(waiter->enter);
+	AN(waiter->fini);
 
 	w = calloc(1, sizeof (struct waiter) + waiter->size);
 	AN(w);
@@ -234,20 +79,10 @@ Waiter_New(waiter_handle_f *func, volatile double *tmo)
 	w->impl = waiter;
 	w->func = func;
 	w->tmo = tmo;
-	w->pipes[0] = w->pipes[1] = -1;
 	VTAILQ_INIT(&w->waithead);
 
 	waiter->init(w);
-	AN(w->impl->pass || w->pipes[1] >= 0);
-
-	Lck_Lock(&wait_mtx);
-	VTAILQ_INSERT_TAIL(&waiters, w, list);
-	nwaiters++;
 
-	/* We assume all waiters either use pipes or don't use pipes */
-	if (w->pipes[1] >= 0 && nwaiters == 1)
-		AZ(pthread_create(&wait_thr, NULL, wait_poker_thread, NULL));
-	Lck_Unlock(&wait_mtx);
 	return (w);
 }
 
@@ -255,47 +90,13 @@ void
 Waiter_Destroy(struct waiter **wp)
 {
 	struct waiter *w;
-	struct waited *wx = NULL;
-	int written;
-	double now;
 
 	AN(wp);
 	w = *wp;
 	*wp = NULL;
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 
-	Lck_Lock(&wait_mtx);
-	VTAILQ_REMOVE(&waiters, w, list);
-	w->dismantle = 1;
-	Lck_Unlock(&wait_mtx);
-
-	if (w->pipes[1] >= 0) {
-		while (1) {
-			written = write(w->pipes[1], &wx, sizeof wx);
-			if (written == sizeof wx)
-				break;
-			(void)usleep(10000);
-		}
-	}
 	AN(w->impl->fini);
 	w->impl->fini(w);
-	now = VTIM_real();
-	while (1) {
-		wx = VTAILQ_FIRST(&w->waithead);
-		if (wx == NULL)
-			break;
-		VTAILQ_REMOVE(&w->waithead, wx, list);
-		if (wx == w->pipe_w)
-			FREE_OBJ(wx);
-		else
-			w->func(wx, WAITER_CLOSE, now);
-	}
 	FREE_OBJ(w);
 }
-
-void
-Waiter_Init(void)
-{
-
-	Lck_New(&wait_mtx, lck_misc);
-}
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index d3cbe4e..467f73f 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -45,80 +45,88 @@ struct vwp {
 #define VWP_MAGIC		0x4b2cc735
 	struct waiter		*waiter;
 
+	int			pipes[2];
+
 	pthread_t		thread;
 	struct pollfd		*pollfd;
-	unsigned		npoll;
-	unsigned		hpoll;
+	struct waited		**idx;
+	size_t			npoll;
+	size_t			hpoll;
 };
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * It would make much more sense to not use two large vectors, but
+ * the poll(2) API forces us to use at least one, so ... KISS.
+ */
 
 static void
-vwp_pollspace(struct vwp *vwp, unsigned fd)
+vwp_extend_pollspace(struct vwp *vwp)
 {
-	struct pollfd *newpollfd = vwp->pollfd;
-	unsigned newnpoll;
-
-	if (fd < vwp->npoll)
-		return;
-	newnpoll = vwp->npoll;
-	if (newnpoll == 0)
-		newnpoll = 1;
-	while (fd >= newnpoll)
-		newnpoll = newnpoll * 2;
-	VSL(SLT_Debug, 0, "Acceptor poll space increased to %u", newnpoll);
-	newpollfd = realloc(newpollfd, newnpoll * sizeof *newpollfd);
-	XXXAN(newpollfd);
-	memset(newpollfd + vwp->npoll, 0,
-	    (newnpoll - vwp->npoll) * sizeof *newpollfd);
-	vwp->pollfd = newpollfd;
-	while (vwp->npoll < newnpoll)
+	size_t inc = (1<<16);
+
+	VSL(SLT_Debug, 0, "Acceptor poll space increased by %zu to %zu",
+	    inc, vwp->npoll + inc);
+
+	vwp->pollfd = realloc(vwp->pollfd,
+	    (vwp->npoll + inc) * sizeof(*vwp->pollfd));
+	AN(vwp->pollfd);
+	memset(vwp->pollfd + vwp->npoll, 0, inc * sizeof(*vwp->pollfd));
+
+	vwp->idx = realloc(vwp->idx, (vwp->npoll + inc) * sizeof(*vwp->idx));
+	AN(vwp->idx);
+	memset(vwp->idx + vwp->npoll, 0, inc * sizeof(*vwp->idx));
+
+	for (; inc > 0; inc--)
 		vwp->pollfd[vwp->npoll++].fd = -1;
-	assert(fd < vwp->npoll);
 }
 
 /*--------------------------------------------------------------------*/
 
-static void __match_proto__(waiter_inject_f)
-vwp_inject(const struct waiter *w, struct waited *wp)
+static void
+vwp_add(struct vwp *vwp, struct waited *w)
 {
-	struct vwp *vwp;
-	int fd;
 
-	CAST_OBJ_NOTNULL(vwp, w->priv, VWP_MAGIC);
-	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
-	fd = wp->fd;
-VSL(SLT_Debug, 0, "POLL Inject %d", fd);
-	assert(fd >= 0);
-	vwp_pollspace(vwp, (unsigned)fd);
-	assert(fd < vwp->npoll);
-	if (vwp->hpoll < fd)
-		vwp->hpoll = fd;
-
-	assert(vwp->pollfd[fd].fd == -1);
-	AZ(vwp->pollfd[fd].events);
-	AZ(vwp->pollfd[fd].revents);
-
-	vwp->pollfd[fd].fd = fd;
-	vwp->pollfd[fd].events = POLLIN;
+	if (vwp->hpoll == vwp->npoll)
+		vwp_extend_pollspace(vwp);
+	assert(vwp->hpoll < vwp->npoll);
+	assert(vwp->pollfd[vwp->hpoll].fd == -1);
+	AZ(vwp->idx[vwp->hpoll]);
+	vwp->pollfd[vwp->hpoll].fd = w->fd;
+	vwp->pollfd[vwp->hpoll].events = POLLIN;
+	vwp->idx[vwp->hpoll] = w;
+	vwp->hpoll++;
 }
 
-static void __match_proto__(waiter_evict_f)
-vwp_evict(const struct waiter *w, struct waited *wp)
+static void
+vwp_del(struct vwp *vwp, int n)
 {
-	struct vwp *vwp;
-	int fd;
+	vwp->hpoll--;
+	if (n != vwp->hpoll) {
+		vwp->pollfd[n] = vwp->pollfd[vwp->hpoll];
+		vwp->idx[n] = vwp->idx[vwp->hpoll];
+	}
+	memset(&vwp->pollfd[vwp->hpoll], 0, sizeof(*vwp->pollfd));
+	vwp->pollfd[vwp->hpoll].fd = -1;
+	vwp->idx[vwp->hpoll] = NULL;
+}
 
-	CAST_OBJ_NOTNULL(vwp, w->priv, VWP_MAGIC);
-	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
-	fd = wp->fd;
-VSL(SLT_Debug, 0, "POLL Evict %d", fd);
-	assert(fd >= 0);
-	assert(fd < vwp->npoll);
-	vwp_pollspace(vwp, (unsigned)fd);
-
-	vwp->pollfd[fd].fd = -1;
-	vwp->pollfd[fd].events = 0;
+/*--------------------------------------------------------------------*/
+
+static void
+vwp_dopipe(struct vwp *vwp)
+{
+	struct waited *w[128];
+	ssize_t ss;
+	int i;
+
+	ss = read(vwp->pipes[0], w, sizeof w);
+	assert(ss > 0);
+	i = 0;
+	while (ss) {
+		CHECK_OBJ_NOTNULL(w[i], WAITED_MAGIC);
+		assert(w[i]->fd > 0);			// no stdin
+		vwp_add(vwp, w[i++]);
+	}
 }
 
 /*--------------------------------------------------------------------*/
@@ -128,50 +136,66 @@ vwp_main(void *priv)
 {
 	int v, v2;
 	struct vwp *vwp;
-	struct waited *sp, *sp2;
+	struct waited *wp;
 	double now, idle;
-	int fd;
+	int i;
 
 	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
 	THR_SetName("cache-poll");
 
-	while (!vwp->waiter->dismantle) {
+	while (1) {
+		// Try to keep the high point as low as possible
 		assert(vwp->hpoll < vwp->npoll);
 		while (vwp->hpoll > 0 && vwp->pollfd[vwp->hpoll].fd == -1)
 			vwp->hpoll--;
+
+		// XXX: sleep on ->tmo
 		v = poll(vwp->pollfd, vwp->hpoll + 1, -1);
 		assert(v >= 0);
 		v2 = v;
 		now = VTIM_real();
 		idle = now - *vwp->waiter->tmo;
-		VTAILQ_FOREACH_SAFE(sp, &vwp->waiter->waithead, list, sp2) {
-			if (v != 0 && v2 == 0)
-				break;
-			CHECK_OBJ_NOTNULL(sp, WAITED_MAGIC);
-			fd = sp->fd;
-			VSL(SLT_Debug, 0,
-			    "POLL Handle %d %x", fd, vwp->pollfd[fd].revents);
-			assert(fd >= 0);
-			assert(fd <= vwp->hpoll);
-			assert(fd < vwp->npoll);
-			assert(vwp->pollfd[fd].fd == fd);
-			if (vwp->pollfd[fd].revents) {
+		i = 1;
+		while (v2 > 0 && i < vwp->hpoll) {
+			wp = vwp->idx[i];
+			CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
+			if (vwp->pollfd[i].revents != 0) {
 				v2--;
-				vwp->pollfd[fd].revents = 0;
-				Wait_Handle(vwp->waiter, sp, WAITER_ACTION,
-				    now);
-			} else if (sp->idle <= idle) {
-				Wait_Handle(vwp->waiter, sp, WAITER_TIMEOUT,
-				    now);
+				assert(wp->fd > 0);
+				assert(wp->fd == vwp->pollfd[i].fd);
+				VSL(SLT_Debug, wp->fd, "POLL Handle %d %x",
+				    wp->fd, vwp->pollfd[i].revents);
+				vwp_del(vwp, i);
+				vwp->waiter->func(wp, WAITER_ACTION, now);
+			} else if (wp->idle <= idle) {
+				vwp_del(vwp, i);
+				vwp->waiter->func(wp, WAITER_TIMEOUT, now);
+			} else {
+				i++;
 			}
 		}
-		Wait_Handle(vwp->waiter, NULL, WAITER_ACTION, now);
+		if (vwp->pollfd[0].revents)
+			vwp_dopipe(vwp);
 	}
 	NEEDLESS_RETURN(NULL);
 }
 
 /*--------------------------------------------------------------------*/
 
+static int __match_proto__(waiter_enter_f)
+vwp_enter(void *priv, struct waited *wp)
+{
+	struct vwp *vwp;
+
+	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
+
+	if (write(vwp->pipes[1], &wp, sizeof wp) != sizeof wp)
+		return (-1);
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
 static void __match_proto__(waiter_init_f)
 vwp_init(struct waiter *w)
 {
@@ -181,9 +205,13 @@ vwp_init(struct waiter *w)
 	vwp = w->priv;
 	INIT_OBJ(vwp, VWP_MAGIC);
 	vwp->waiter = w;
+	AZ(pipe(vwp->pipes));
+	// XXX: set write pipe non-blocking
 
-	vwp_pollspace(vwp, 256);
-	Wait_UsePipe(w);
+	vwp_extend_pollspace(vwp);
+	vwp->pollfd[0].fd = vwp->pipes[0];
+	vwp->pollfd[0].events = POLLIN;
+	vwp->hpoll = 1;
 	AZ(pthread_create(&vwp->thread, NULL, vwp_main, vwp));
 }
 
@@ -196,6 +224,9 @@ vwp_fini(struct waiter *w)
 	void *vp;
 
 	CAST_OBJ_NOTNULL(vwp, w->priv, VWP_MAGIC);
+	vp = NULL;
+	// XXX: set write pipe blocking
+	assert(write(vwp->pipes[1], &vp, sizeof vp) == sizeof vp);
 	AZ(pthread_join(vwp->thread, &vp));
 	free(vwp->pollfd);
 }
@@ -206,7 +237,6 @@ const struct waiter_impl waiter_poll = {
 	.name =		"poll",
 	.init =		vwp_init,
 	.fini =		vwp_fini,
-	.inject =	vwp_inject,
-	.evict =	vwp_evict,
+	.enter =	vwp_enter,
 	.size =		sizeof(struct vwp),
 };
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
index db69e26..5b5620b 100644
--- a/bin/varnishd/waiter/waiter.h
+++ b/bin/varnishd/waiter/waiter.h
@@ -61,7 +61,6 @@ int Wait_Enter(const struct waiter *, struct waited *);
 struct waiter *Waiter_New(waiter_handle_f *, volatile double *timeout);
 void Waiter_Destroy(struct waiter **);
 const char *Waiter_GetName(void);
-void Waiter_Init(void);
 
 /* mgt_waiter.c */
 int Wait_Argument(struct vsb *vsb, const char *arg);
diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h
index f21e0ae..ed369c4 100644
--- a/bin/varnishd/waiter/waiter_priv.h
+++ b/bin/varnishd/waiter/waiter_priv.h
@@ -33,27 +33,25 @@ struct waited;
 
 struct waiter {
 	unsigned			magic;
-	#define WAITER_MAGIC		0x17c399db
+#define WAITER_MAGIC			0x17c399db
 	const struct waiter_impl	*impl;
 	VTAILQ_ENTRY(waiter)		list;
+	VTAILQ_HEAD(,waited)		waithead;
+
 	int				dismantle;
 
 	waiter_handle_f *		func;
 
-	int				pipes[2];
-	struct waited			*pipe_w;
 	double				next_idle;
-	int				do_pipe;
 
 	volatile double			*tmo;
-	VTAILQ_HEAD(,waited)		waithead;
 
 	void				*priv;
 };
 
 typedef void waiter_init_f(struct waiter *);
 typedef void waiter_fini_f(struct waiter *);
-typedef int waiter_pass_f(void *priv, struct waited *);
+typedef int waiter_enter_f(void *priv, struct waited *);
 typedef void waiter_inject_f(const struct waiter *, struct waited *);
 typedef void waiter_evict_f(const struct waiter *, struct waited *);
 
@@ -61,16 +59,11 @@ struct waiter_impl {
 	const char		*name;
 	waiter_init_f		*init;
 	waiter_fini_f		*fini;
-	waiter_pass_f		*pass;
+	waiter_enter_f		*enter;
 	waiter_inject_f		*inject;
-	waiter_evict_f		*evict;
 	size_t			size;
 };
 
-/* cache_waiter.c */
-void Wait_Handle(struct waiter *, struct waited *, enum wait_event, double now);
-void Wait_UsePipe(struct waiter *w);
-
 /* mgt_waiter.c */
 extern struct waiter_impl const * waiter;
 



From phk at FreeBSD.org  Thu May 21 16:48:51 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 18:48:51 +0200
Subject: [master] f2c1e64 Patch from dridi to replace CLI with mutext for
 locking backend config.
Message-ID: <E1YvTeN-00056w-I3@project.varnish-software.com>


commit f2c1e642df9c0a71d01803ebb270f6a918e56cfc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 09:28:09 2015 +0000

    Patch from dridi to replace CLI with mutext for locking backend config.

diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index 08b58d3..b917618 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -46,11 +46,8 @@
 #include "vrt.h"
 #include "vtim.h"
 
-/*
- * The list of backends is not locked, it is only ever accessed from
- * the CLI thread, so there is no need.
- */
 static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends);
+static struct lock backends_mtx;
 
 /*--------------------------------------------------------------------
  */
@@ -59,10 +56,13 @@ void
 VBE_DeleteBackend(struct backend *b)
 {
 
-	ASSERT_CLI();
 	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
 
+	Lck_Lock(&backends_mtx);
 	VTAILQ_REMOVE(&backends, b, list);
+	VSC_C_main->n_backend--;
+	Lck_Unlock(&backends_mtx);
+
 	free(b->ipv4);
 	free(b->ipv6);
 	free(b->display_name);
@@ -70,7 +70,6 @@ VBE_DeleteBackend(struct backend *b)
 	VBT_Rel(&b->tcp_pool);
 	Lck_Delete(&b->mtx);
 	FREE_OBJ(b);
-	VSC_C_main->n_backend--;
 }
 
 /*--------------------------------------------------------------------
@@ -85,7 +84,6 @@ VBE_AddBackend(const char *vcl, const struct vrt_backend *vb)
 	struct backend *b;
 	char buf[128];
 
-	ASSERT_CLI();
 	AN(vb->vcl_name);
 	assert(vb->ipv4_suckaddr != NULL || vb->ipv6_suckaddr != NULL);
 
@@ -118,8 +116,10 @@ VBE_AddBackend(const char *vcl, const struct vrt_backend *vb)
 	b->health_changed = VTIM_real();
 	b->admin_health = ah_probe;
 
+	Lck_Lock(&backends_mtx);
 	VTAILQ_INSERT_TAIL(&backends, b, list);
 	VSC_C_main->n_backend++;
+	Lck_Unlock(&backends_mtx);
 	return (b);
 }
 
@@ -176,6 +176,7 @@ backend_find(struct cli *cli, const char *matcher, bf_func *func, void *priv)
 		VSB_printf(vsb, "%s.%s", vcc->loaded_name, matcher);
 	}
 	AZ(VSB_finish(vsb));
+	Lck_Lock(&backends_mtx);
 	VTAILQ_FOREACH(b, &backends, list) {
 		if (fnmatch(VSB_data(vsb), b->display_name, 0))
 			continue;
@@ -186,6 +187,7 @@ backend_find(struct cli *cli, const char *matcher, bf_func *func, void *priv)
 			break;
 		}
 	}
+	Lck_Unlock(&backends_mtx);
 	VSB_delete(vsb);
 	return (found);
 }
@@ -313,4 +315,5 @@ VBE_InitCfg(void)
 {
 
 	CLI_AddFuncs(backend_cmds);
+	Lck_New(&backends_mtx, lck_vbe);
 }
diff --git a/include/tbl/locks.h b/include/tbl/locks.h
index e4b4914..010de43 100644
--- a/include/tbl/locks.h
+++ b/include/tbl/locks.h
@@ -46,6 +46,7 @@ LOCK(exp)
 LOCK(lru)
 LOCK(cli)
 LOCK(ban)
+LOCK(vbe)
 LOCK(vbp)
 LOCK(backend)
 LOCK(vcapace)



From phk at FreeBSD.org  Thu May 21 16:48:51 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 18:48:51 +0200
Subject: [master] 385930c Rework the backend/waiter stuff.
Message-ID: <E1YvTeN-000575-J5@project.varnish-software.com>


commit 385930c383eae465e661673f165bd22d6836ace5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 15:04:00 2015 +0000

    Rework the backend/waiter stuff.
    
    Instead of trying to steal vbc's away from the waiter, send the
    request and wait for the waiter to hand the vbc back to us.
    
    All waiters but poll still disabled ... coming up next.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 858ade7..d451943 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -78,7 +78,7 @@ VBE_Healthy(const struct backend *backend, double *changed)
  */
 
 static int __match_proto__(vdi_getfd_f)
-vbe_dir_getfd(const struct director *d, struct busyobj *bo)
+vbe_dir_getfd(struct worker *wrk, const struct director *d, struct busyobj *bo)
 {
 	struct vbc *vc;
 	struct backend *bp;
@@ -87,6 +87,7 @@ vbe_dir_getfd(const struct director *d, struct busyobj *bo)
 	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
 	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
@@ -112,7 +113,7 @@ vbe_dir_getfd(const struct director *d, struct busyobj *bo)
 		return (-1);
 
 	FIND_TMO(connect_timeout, tmod, bo, vrt);
-	vc = VBT_Get(bp->tcp_pool, tmod);
+	vc = VBT_Get(bp->tcp_pool, tmod, bp, wrk);
 	if (vc == NULL) {
 		// XXX: Per backend stats ?
 		VSC_C_main->backend_fail++;
@@ -197,7 +198,7 @@ static int __match_proto__(vdi_gethdrs_f)
 vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
     struct busyobj *bo)
 {
-	int i;
+	int i, extrachance = 0;
 	const struct vrt_backend *vrt;
 
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
@@ -205,12 +206,14 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CAST_OBJ_NOTNULL(vrt, d->priv2, VRT_BACKEND_MAGIC);
 
-	i = vbe_dir_getfd(d, bo);
+	i = vbe_dir_getfd(wrk, d, bo);
 	if (i < 0) {
 		VSLb(bo->vsl, SLT_FetchError, "no backend connection");
 		return (-1);
 	}
 	AN(bo->htc);
+	if (bo->htc->vbc->state == VBC_STATE_STOLEN)
+		extrachance = 1;
 
 	i = V1F_fetch_hdr(wrk, bo, vrt->hosthdr);
 	/*
@@ -218,12 +221,12 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 	 * that the backend closed it before we get a request to it.
 	 * Do a single retry in that case.
 	 */
-	if (i == 1 && bo->htc->vbc->recycled) {
+	if (i == 1 && extrachance) {
 		vbe_dir_finish(d, wrk, bo);
 		AZ(bo->htc);
 		VSC_C_main->backend_retry++;
 		bo->doclose = SC_NULL;
-		i = vbe_dir_getfd(d, bo);
+		i = vbe_dir_getfd(wrk, d, bo);
 		if (i < 0) {
 			VSLb(bo->vsl, SLT_FetchError, "no backend connection");
 			bo->htc = NULL;
@@ -281,13 +284,15 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	i = vbe_dir_getfd(d, bo);
+	i = vbe_dir_getfd(req->wrk, d, bo);
 	if (i < 0) {
 		VSLb(bo->vsl, SLT_FetchError, "no backend connection");
+		SES_Close(req->sp, SC_RX_TIMEOUT);
 		return;
+	} else {
+		V1P_Process(req, bo, i);
+		vbe_dir_finish(d, req->wrk, bo);
 	}
-	V1P_Process(req, bo, i);
-	vbe_dir_finish(d, bo->wrk, bo);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 38a979f..7fdc55b 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -88,21 +88,19 @@ struct backend {
 struct vbc {
 	unsigned		magic;
 #define VBC_MAGIC		0x0c5e6592
-	VTAILQ_ENTRY(vbc)	list;
 	int			fd;
+	VTAILQ_ENTRY(vbc)	list;
 	const struct suckaddr	*addr;
-	uint8_t			recycled;
 	uint8_t			state;
 #define VBC_STATE_AVAIL		(1<<0)
 #define VBC_STATE_USED		(1<<1)
 #define VBC_STATE_STOLEN	(1<<2)
 #define VBC_STATE_CLEANUP	(1<<3)
-	uint8_t			in_waiter;
-	uint8_t			have_been_in_waiter;
 	struct waited		waited[1];
 	struct tcp_pool		*tcp_pool;
 
 	struct backend		*backend;
+	struct worker		*wrk;
 };
 
 /* cache_backend_cfg.c */
@@ -123,6 +121,7 @@ void VBT_Rel(struct tcp_pool **tpp);
 int VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa);
 void VBT_Recycle(struct tcp_pool *tp, struct vbc **vbc);
 void VBT_Close(struct tcp_pool *tp, struct vbc **vbc);
-struct vbc *VBT_Get(struct tcp_pool *tp, double tmo);
-
+struct vbc *VBT_Get(struct tcp_pool *, double tmo, struct backend *,
+    struct worker *);
+void VBT_Wait(struct worker *wrk, const struct vbc *vbc);
 
diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index ef037c9..44b6351 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -44,7 +44,6 @@
 #include "waiter/waiter.h"
 #include "vtim.h"
 
-
 struct tcp_pool {
 	unsigned		magic;
 #define TCP_POOL_MAGIC		0x28b0e42a
@@ -89,24 +88,14 @@ tcp_handle(struct waited *w, enum wait_event ev, double now)
 	tp = vbc->tcp_pool;
 
 	Lck_Lock(&tp->mtx);
-	VSL(SLT_Debug, 0,
-	    "------> Handler fd %d in_w %d state 0x%x ev %d have_been %d",
-	    vbc->fd, vbc->in_waiter, vbc->state, ev, vbc->have_been_in_waiter);
-	AN(vbc->in_waiter);
 
 	switch(vbc->state) {
 	case VBC_STATE_STOLEN:
-		vbc->state = VBC_STATE_AVAIL;
+		vbc->state = VBC_STATE_USED;
 		VTAILQ_REMOVE(&tp->connlist, vbc, list);
-		if (Wait_Enter(tp->waiter, vbc->waited)) {
-			VSL(SLT_Debug, 0,
-			    "------> Handler stolen -> re-wait failed");
-			VTCP_close(&vbc->fd);
-			tp->n_conn--;
-			FREE_OBJ(vbc);
-		} else {
-			VTAILQ_INSERT_HEAD(&tp->connlist, vbc, list);
-		}
+		CHECK_OBJ_NOTNULL(vbc->backend, BACKEND_MAGIC);
+		CHECK_OBJ_NOTNULL(vbc->wrk, WORKER_MAGIC);
+		AZ(pthread_cond_signal(&vbc->wrk->cond));
 		break;
 	case VBC_STATE_AVAIL:
 		VTCP_close(&vbc->fd);
@@ -114,10 +103,6 @@ tcp_handle(struct waited *w, enum wait_event ev, double now)
 		tp->n_conn--;
 		FREE_OBJ(vbc);
 		break;
-	case VBC_STATE_USED:
-		vbc->in_waiter = 0;
-		vbc->have_been_in_waiter = 1;
-		break;
 	case VBC_STATE_CLEANUP:
 		VTCP_close(&vbc->fd);
 		tp->n_kill--;
@@ -208,16 +193,11 @@ VBT_Rel(struct tcp_pool **tpp)
 	VTAILQ_FOREACH_SAFE(vbc, &tp->connlist, list, vbc2) {
 		VTAILQ_REMOVE(&tp->connlist, vbc, list);
 		tp->n_conn--;
-		if (vbc->in_waiter) {
-			vbc->state = VBC_STATE_CLEANUP;
-			shutdown(vbc->fd, SHUT_WR);
-			VTAILQ_INSERT_TAIL(&tp->killlist, vbc, list);
-			tp->n_kill++;
-		} else {
-			VTCP_close(&vbc->fd);
-			memset(vbc, 0x22, sizeof *vbc);
-			free(vbc);
-		}
+		assert(vbc->state == VBC_STATE_AVAIL);
+		vbc->state = VBC_STATE_CLEANUP;
+		(void)shutdown(vbc->fd, SHUT_WR);
+		VTAILQ_INSERT_TAIL(&tp->killlist, vbc, list);
+		tp->n_kill++;
 	}
 	while (tp->n_kill) {
 		Lck_Unlock(&tp->mtx);
@@ -279,39 +259,30 @@ VBT_Recycle(struct tcp_pool *tp, struct vbc **vbcp)
 
 	assert(vbc->state == VBC_STATE_USED);
 	assert(vbc->fd > 0);
+	AZ(vbc->backend);
 
 	Lck_Lock(&tp->mtx);
 	tp->n_used--;
 
-	VSL(SLT_Debug, 0, "------> Recycle fd %d in_w %d",
-	    vbc->fd, vbc->in_waiter);
-
-	if (!vbc->in_waiter) {
-		vbc->in_waiter = 1;
-		vbc->waited->ptr = vbc;
-		vbc->waited->fd = vbc->fd;
-		vbc->waited->idle = VTIM_real();
-		vbc->state = VBC_STATE_AVAIL;
-		VSL(SLT_Debug, 0, "------> Recycle fd %d Wait_Enter", vbc->fd);
-		if (Wait_Enter(tp->waiter, vbc->waited)) {
-			VTCP_close(&vbc->fd);
-			memset(vbc, 0x33, sizeof *vbc);
-			free(vbc);
-			vbc = NULL;
-		} else {
-			VTAILQ_INSERT_HEAD(&tp->connlist, vbc, list);
-		}
-		i = 1;
+	vbc->waited->ptr = vbc;
+	vbc->waited->fd = vbc->fd;
+	vbc->waited->idle = VTIM_real();
+	vbc->state = VBC_STATE_AVAIL;
+	if (Wait_Enter(tp->waiter, vbc->waited)) {
+		VTCP_close(&vbc->fd);
+		memset(vbc, 0x33, sizeof *vbc);
+		free(vbc);
+		// XXX: stats
+		vbc = NULL;
 	} else {
-		vbc->state = VBC_STATE_STOLEN;
-		VTAILQ_INSERT_TAIL(&tp->connlist, vbc, list);
+		VTAILQ_INSERT_HEAD(&tp->connlist, vbc, list);
+		i++;
 	}
 
-	if (vbc != NULL) {
+	if (vbc != NULL)
 		tp->n_conn++;
-		vbc->recycled = 1;
-	}
 	Lck_Unlock(&tp->mtx);
+
 	if (i && DO_DEBUG(DBG_VTC_MODE)) {
 		/*
 		 * In varnishtest we do not have the luxury of using
@@ -346,17 +317,17 @@ VBT_Close(struct tcp_pool *tp, struct vbc **vbcp)
 
 	assert(vbc->state == VBC_STATE_USED);
 	assert(vbc->fd > 0);
-	VSL(SLT_Debug, 0, "------> Close fd %d in_w %d",
-	    vbc->fd, vbc->in_waiter);
+	AZ(vbc->backend);
 
 	Lck_Lock(&tp->mtx);
 	tp->n_used--;
-	if (vbc->in_waiter) {
-		shutdown(vbc->fd, SHUT_WR);
+	if (vbc->state == VBC_STATE_STOLEN) {
+		(void)shutdown(vbc->fd, SHUT_WR);
 		vbc->state = VBC_STATE_CLEANUP;
 		VTAILQ_INSERT_HEAD(&tp->killlist, vbc, list);
 		tp->n_kill++;
 	} else {
+		assert(vbc->state == VBC_STATE_USED);
 		VTCP_close(&vbc->fd);
 		memset(vbc, 0x44, sizeof *vbc);
 		free(vbc);
@@ -369,27 +340,29 @@ VBT_Close(struct tcp_pool *tp, struct vbc **vbcp)
  */
 
 struct vbc *
-VBT_Get(struct tcp_pool *tp, double tmo)
+VBT_Get(struct tcp_pool *tp, double tmo, struct backend *be, struct worker *wrk)
 {
 	struct vbc *vbc;
 
 	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
+	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	Lck_Lock(&tp->mtx);
 	vbc = VTAILQ_FIRST(&tp->connlist);
-	if (vbc != NULL) {
-		CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC);
-
-		VSL(SLT_Debug, 0, "------> Steal fd %d state 0x%x",
-		    vbc->fd, vbc->state);
-		assert(vbc->state == VBC_STATE_AVAIL ||
-		    vbc->state == VBC_STATE_STOLEN);
-
+	CHECK_OBJ_ORNULL(vbc, VBC_MAGIC);
+	if (vbc == NULL || vbc->backend != NULL)
+		vbc = NULL;
+	else {
+		assert(vbc->tcp_pool == tp);
+		assert(vbc->state == VBC_STATE_AVAIL);
 		VTAILQ_REMOVE(&tp->connlist, vbc, list);
+		VTAILQ_INSERT_TAIL(&tp->connlist, vbc, list);
 		tp->n_conn--;
 		VSC_C_main->backend_reuse += 1;
-		vbc->state = VBC_STATE_USED;
-		assert(vbc->tcp_pool == tp);
+		vbc->state = VBC_STATE_STOLEN;
+		vbc->backend = be;
+		vbc->wrk = wrk;
 	}
 	tp->n_used++;			// Opening mostly works
 	Lck_Unlock(&tp->mtx);
@@ -402,15 +375,33 @@ VBT_Get(struct tcp_pool *tp, double tmo)
 	INIT_OBJ(vbc->waited, WAITED_MAGIC);
 	vbc->state = VBC_STATE_USED;
 	vbc->tcp_pool = tp;
+	vbc->backend = be;
 	vbc->fd = VBT_Open(tp, tmo, &vbc->addr);
 	if (vbc->fd < 0)
 		FREE_OBJ(vbc);
 	if (vbc == NULL) {
-		VSL(SLT_Debug, 0, "------> No new fd");
 		Lck_Lock(&tp->mtx);
 		tp->n_used--;		// Nope, didn't work after all.
 		Lck_Unlock(&tp->mtx);
-	} else
-		VSL(SLT_Debug, 0, "------> New fd %d", vbc->fd);
+	}
 	return (vbc);
 }
+
+/*--------------------------------------------------------------------
+ */
+
+void
+VBT_Wait(struct worker *wrk, const struct vbc *vbc)
+{
+	struct tcp_pool *tp;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC);
+	tp = vbc->tcp_pool;
+	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
+	assert(vbc->wrk == wrk);
+	Lck_Lock(&tp->mtx);
+	while (vbc->state == VBC_STATE_STOLEN)
+		AZ(Lck_CondWait(&wrk->cond, &tp->mtx, 0));
+	Lck_Unlock(&tp->mtx);
+}
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 7b108d2..23fc6b9 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -37,6 +37,7 @@
 
 #include "hash/hash_slinger.h"
 
+#include "cache/cache_backend.h"
 #include "cache/cache_director.h"
 #include "vcli_priv.h"
 #include "vtcp.h"
@@ -144,6 +145,11 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, const char *def_host)
 
 	/* Receive response */
 
+	if (htc->vbc->state != VBC_STATE_USED)
+		VBT_Wait(wrk, htc->vbc);
+
+	assert(htc->vbc->state == VBC_STATE_USED);
+
 	SES_RxInit(htc, bo->ws, cache_param->http_resp_size,
 	    cache_param->http_resp_hdr_len);
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index ac1fb6e..818f88f 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -105,6 +105,7 @@ V1P_Process(struct req *req, struct busyobj *bo, int fd)
 	wrk = req->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	assert(fd > 0);
 
 	req->res_mode = RES_PIPE;
 
@@ -112,12 +113,8 @@ V1P_Process(struct req *req, struct busyobj *bo, int fd)
 	acct_pipe.req = req->acct.req_hdrbytes;
 	req->acct.req_hdrbytes = 0;
 
-	if (fd < 0) {
-		pipecharge(req, &acct_pipe, NULL);
-		SES_Close(req->sp, SC_OVERLOAD);
-		return;
-	}
 	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->htc->vbc, VBC_MAGIC);
 	bo->wrk = req->wrk;
 	bo->director_state = DIR_S_BODY;
 	(void)VTCP_blocking(fd);
@@ -134,6 +131,9 @@ V1P_Process(struct req *req, struct busyobj *bo, int fd)
 	VSLb_ts_req(req, "Pipe", W_TIM_real(wrk));
 
 	if (i == 0) {
+		if (bo->htc->vbc->state == VBC_STATE_STOLEN)
+			VBT_Wait(req->wrk, bo->htc->vbc);
+
 		memset(fds, 0, sizeof fds);
 		fds[0].fd = fd;
 		fds[0].events = POLLIN | POLLERR;
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index 467f73f..bb87b17 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -62,7 +62,16 @@ struct vwp {
 static void
 vwp_extend_pollspace(struct vwp *vwp)
 {
-	size_t inc = (1<<16);
+	size_t inc;
+
+	if (vwp->npoll < (1<<12))
+		inc = (1<<10);
+	else if (vwp->npoll < (1<<14))
+		inc = (1<<12);
+	else if (vwp->npoll < (1<<16))
+		inc = (1<<14);
+	else
+		inc = (1<<16);
 
 	VSL(SLT_Debug, 0, "Acceptor poll space increased by %zu to %zu",
 	    inc, vwp->npoll + inc);
@@ -86,6 +95,8 @@ static void
 vwp_add(struct vwp *vwp, struct waited *w)
 {
 
+	CHECK_OBJ_NOTNULL(w, WAITED_MAGIC);
+	CHECK_OBJ_NOTNULL(vwp, VWP_MAGIC);
 	if (vwp->hpoll == vwp->npoll)
 		vwp_extend_pollspace(vwp);
 	assert(vwp->hpoll < vwp->npoll);
@@ -123,9 +134,15 @@ vwp_dopipe(struct vwp *vwp)
 	assert(ss > 0);
 	i = 0;
 	while (ss) {
+		if (w[i] == NULL) {
+			assert(ss == sizeof w[0]);
+			assert(vwp->hpoll == 1);
+			pthread_exit(NULL);
+		}
 		CHECK_OBJ_NOTNULL(w[i], WAITED_MAGIC);
 		assert(w[i]->fd > 0);			// no stdin
 		vwp_add(vwp, w[i++]);
+		ss -= sizeof w[0];
 	}
 }
 
@@ -134,47 +151,49 @@ vwp_dopipe(struct vwp *vwp)
 static void *
 vwp_main(void *priv)
 {
-	int v, v2;
+	int v;
 	struct vwp *vwp;
 	struct waited *wp;
 	double now, idle;
-	int i;
+	int i, dopipe;
 
-	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
 	THR_SetName("cache-poll");
+	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
 
 	while (1) {
-		// Try to keep the high point as low as possible
-		assert(vwp->hpoll < vwp->npoll);
-		while (vwp->hpoll > 0 && vwp->pollfd[vwp->hpoll].fd == -1)
-			vwp->hpoll--;
-
-		// XXX: sleep on ->tmo
-		v = poll(vwp->pollfd, vwp->hpoll + 1, -1);
+		v = poll(vwp->pollfd, vwp->hpoll,
+		    (int)floor(1e3 * *vwp->waiter->tmo));
 		assert(v >= 0);
-		v2 = v;
+		if (v == 0)
+			v = vwp->hpoll;
 		now = VTIM_real();
 		idle = now - *vwp->waiter->tmo;
-		i = 1;
-		while (v2 > 0 && i < vwp->hpoll) {
+		i = 0;
+		dopipe = 0;
+		while (v > 0 && i < vwp->hpoll) {
+			if (vwp->pollfd[i].revents)
+				v--;
+			if (vwp->pollfd[i].fd == vwp->pipes[0]) {
+				if (vwp->pollfd[i].revents)
+					dopipe = 1;
+				i++;
+				continue;
+			}
 			wp = vwp->idx[i];
 			CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
-			if (vwp->pollfd[i].revents != 0) {
-				v2--;
+			if (wp->idle <= idle) {
+				vwp->waiter->func(wp, WAITER_TIMEOUT, now);
+				vwp_del(vwp, i);
+			} else if (vwp->pollfd[i].revents & POLLIN) {
 				assert(wp->fd > 0);
 				assert(wp->fd == vwp->pollfd[i].fd);
-				VSL(SLT_Debug, wp->fd, "POLL Handle %d %x",
-				    wp->fd, vwp->pollfd[i].revents);
-				vwp_del(vwp, i);
 				vwp->waiter->func(wp, WAITER_ACTION, now);
-			} else if (wp->idle <= idle) {
 				vwp_del(vwp, i);
-				vwp->waiter->func(wp, WAITER_TIMEOUT, now);
 			} else {
 				i++;
 			}
 		}
-		if (vwp->pollfd[0].revents)
+		if (dopipe)
 			vwp_dopipe(vwp);
 	}
 	NEEDLESS_RETURN(NULL);
@@ -208,10 +227,10 @@ vwp_init(struct waiter *w)
 	AZ(pipe(vwp->pipes));
 	// XXX: set write pipe non-blocking
 
+	vwp->hpoll = 1;
 	vwp_extend_pollspace(vwp);
 	vwp->pollfd[0].fd = vwp->pipes[0];
 	vwp->pollfd[0].events = POLLIN;
-	vwp->hpoll = 1;
 	AZ(pthread_create(&vwp->thread, NULL, vwp_main, vwp));
 }
 
@@ -228,7 +247,10 @@ vwp_fini(struct waiter *w)
 	// XXX: set write pipe blocking
 	assert(write(vwp->pipes[1], &vp, sizeof vp) == sizeof vp);
 	AZ(pthread_join(vwp->thread, &vp));
+	AZ(close(vwp->pipes[0]));
+	AZ(close(vwp->pipes[1]));
 	free(vwp->pollfd);
+	free(vwp->idx);
 }
 
 /*--------------------------------------------------------------------*/



From phk at FreeBSD.org  Thu May 21 16:48:51 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 18:48:51 +0200
Subject: [master] 64fbc76 Bring back the kqueue waiter.
Message-ID: <E1YvTeN-00057F-K6@project.varnish-software.com>


commit 64fbc76108d77a601957165317dba1163d70fa6b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 16:41:26 2015 +0000

    Bring back the kqueue waiter.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 89d4cd9..1a0dc03 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -376,6 +376,7 @@ struct waited {
 	int			fd;
 	void			*ptr;
 	double			idle;
+	VTAILQ_ENTRY(waited)	list;
 };
 
 /* Stored object -----------------------------------------------------
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index c28b2de..f5452ac 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -26,12 +26,8 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * XXX: We need to pass sessions back into the event engine when they are
- * reused.  Not sure what the most efficient way is for that.  For now
- * write the session pointer to a pipe which the event engine monitors.
  */
 
-#if 0
 #include "config.h"
 
 #if defined(HAVE_KQUEUE)
@@ -39,6 +35,7 @@
 #include <sys/types.h>
 #include <sys/event.h>
 
+#include <errno.h>
 #include <stdlib.h>
 
 #include "cache/cache.h"
@@ -47,95 +44,18 @@
 #include "waiter/waiter_priv.h"
 #include "vtim.h"
 
-#define NKEV	100
+#define NKEV	256
 
 struct vwk {
 	unsigned		magic;
 #define VWK_MAGIC		0x1cc2acc2
+	int			kq;
 	struct waiter		*waiter;
-
 	pthread_t		thread;
-	int			kq;
-	struct kevent		ki[NKEV];
-	unsigned		nki;
-};
-
-/*--------------------------------------------------------------------*/
-
-static void
-vwk_kq_flush(struct vwk *vwk)
-{
-	int i;
-
-	if (vwk->nki == 0)
-		return;
-	i = kevent(vwk->kq, vwk->ki, vwk->nki, NULL, 0, NULL);
-	AZ(i);
-	vwk->nki = 0;
-}
-
-static void
-vwk_kq_sess(struct vwk *vwk, struct waited *sp, short arm)
-{
-
-	CHECK_OBJ_NOTNULL(sp, WAITED_MAGIC);
-	assert(sp->fd >= 0);
-	EV_SET(&vwk->ki[vwk->nki], sp->fd, EVFILT_READ, arm, 0, 0, sp);
-	if (++vwk->nki == NKEV)
-		vwk_kq_flush(vwk);
-}
-
-/*--------------------------------------------------------------------*/
 
-static void
-vwk_inject(const struct waiter *w, struct waited *wp)
-{
-	struct vwk *vwk;
-
-	CAST_OBJ_NOTNULL(vwk, w->priv, VWK_MAGIC);
-	if (wp == w->pipe_w)
-		vwk_kq_sess(vwk, wp, EV_ADD);
-	else
-		vwk_kq_sess(vwk, wp, EV_ADD | EV_ONESHOT);
-}
-
-#if 0
-static void
-vwk_evict(const struct waiter *w, struct waited *wp)
-{
-	struct vwk *vwk;
-
-	CAST_OBJ_NOTNULL(vwk, w->priv, VWK_MAGIC);
-	vwk_kq_sess(vwk, wp, EV_DELETE);
-}
-#endif
-
-/*--------------------------------------------------------------------*/
-
-static void
-vwk_sess_ev(const struct vwk *vwk, const struct kevent *kp, double now)
-{
-	struct waited *sp;
-	double idle;
-
-	AN(kp->udata);
-	CAST_OBJ_NOTNULL(sp, kp->udata, WAITED_MAGIC);
-
-	idle = now - *vwk->waiter->tmo;
-
-	if (sp->idle <= idle) {
-		Wait_Handle(vwk->waiter, sp, WAITER_TIMEOUT, now);
-	} else if (kp->flags & EV_EOF) {
-		Wait_Handle(vwk->waiter, sp, WAITER_REMCLOSE, now);
-	} else {
-		if (kp->data == 0)
-			VSL(SLT_Debug, 0,
-			    "KQR d %ju filter %d data %jd flags 0x%x idle %g",
-			    (uintmax_t)kp->ident, kp->filter,
-			    (intmax_t)kp->data, kp->flags, sp->idle - idle);
-		Wait_Handle(vwk->waiter, sp, WAITER_ACTION, now);
-	}
-}
+	VTAILQ_HEAD(,waited)	list;
+	struct lock		mtx;
+};
 
 /*--------------------------------------------------------------------*/
 
@@ -145,30 +65,77 @@ vwk_thread(void *priv)
 	struct vwk *vwk;
 	struct kevent ke[NKEV], *kp;
 	int j, n;
-	double now;
+	double now, idle, last_idle;
+	struct timespec ts;
+	struct waited *wp, *wp2;
 
 	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
 	THR_SetName("cache-kqueue");
 
-	vwk_kq_flush(vwk);
-
-	vwk->nki = 0;
-	while (!vwk->waiter->dismantle) {
-		n = kevent(vwk->kq, vwk->ki, vwk->nki, ke, NKEV, NULL);
+	last_idle = 0.0;
+	while (1) {
+		now = .3 * *vwk->waiter->tmo;
+		ts.tv_sec = (time_t)floor(now);
+		ts.tv_nsec = (long)(1e9 * (now - ts.tv_sec));
+		n = kevent(vwk->kq, NULL, 0, ke, NKEV, &ts);
+		if (n < 0 && errno == EBADF)
+			break;
 		assert(n <= NKEV);
-		if (n == 0) {
-			/* This happens on OSX in m00011.vtc */
-			(void)usleep(10000);
-		}
-		vwk->nki = 0;
 		now = VTIM_real();
+		idle = now - *vwk->waiter->tmo;
 		for (kp = ke, j = 0; j < n; j++, kp++) {
 			assert(kp->filter == EVFILT_READ);
-			vwk_sess_ev(vwk, kp, now);
+			CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC);
+			Lck_Lock(&vwk->mtx);
+			VTAILQ_REMOVE(&vwk->list, wp, list);
+			Lck_Unlock(&vwk->mtx);
+			if (wp->idle <= idle)
+				vwk->waiter->func(wp, WAITER_TIMEOUT, now);
+			else if (kp->flags & EV_EOF)
+				vwk->waiter->func(wp, WAITER_REMCLOSE, now);
+			else
+				vwk->waiter->func(wp, WAITER_ACTION, now);
+		}
+		if (now - last_idle > .3 * *vwk->waiter->tmo) {
+			last_idle = now;
+			n = 0;
+			Lck_Lock(&vwk->mtx);
+			VTAILQ_FOREACH_SAFE(wp, &vwk->list, list, wp2) {
+				if (wp->idle > idle)
+					continue;
+				EV_SET(ke + n, wp->fd,
+				    EVFILT_READ, EV_DELETE, 0, 0, wp);
+				if (++n == NKEV)
+					break;
+			}
+			if (n > 0)
+				AZ(kevent(vwk->kq, ke, n, NULL, 0, NULL));
+			for (j = 0; j < n; j++) {
+				CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC);
+				VTAILQ_REMOVE(&vwk->list, wp, list);
+				vwk->waiter->func(wp, WAITER_TIMEOUT, now);
+			}
+			Lck_Unlock(&vwk->mtx);
 		}
-		Wait_Handle(vwk->waiter, NULL, WAITER_ACTION, now);
 	}
-	NEEDLESS_RETURN(NULL);
+	return(NULL);
+}
+
+/*--------------------------------------------------------------------*/
+
+static int __match_proto__(waiter_enter_f)
+vwk_enter(void *priv, struct waited *wp)
+{
+	struct vwk *vwk;
+	struct kevent ke;
+
+	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
+	EV_SET(&ke, wp->fd, EVFILT_READ, EV_ADD|EV_ONESHOT, 0, 0, wp);
+	Lck_Lock(&vwk->mtx);
+	VTAILQ_INSERT_TAIL(&vwk->list, wp, list);
+	AZ(kevent(vwk->kq, &ke, 1, NULL, 0, NULL));
+	Lck_Unlock(&vwk->mtx);
+	return(0);
 }
 
 /*--------------------------------------------------------------------*/
@@ -185,13 +152,16 @@ vwk_init(struct waiter *w)
 
 	vwk->kq = kqueue();
 	assert(vwk->kq >= 0);
-
-	Wait_UsePipe(w);
+	VTAILQ_INIT(&vwk->list);
+	Lck_New(&vwk->mtx, lck_misc);
 
 	AZ(pthread_create(&vwk->thread, NULL, vwk_thread, vwk));
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * It is the callers responsibility to trigger all fd's waited on to
+ * fail somehow.
+ */
 
 static void __match_proto__(waiter_fini_f)
 vwk_fini(struct waiter *w)
@@ -200,8 +170,16 @@ vwk_fini(struct waiter *w)
 	void *vp;
 
 	CAST_OBJ_NOTNULL(vwk, w->priv, VWK_MAGIC);
-	AZ(pthread_join(vwk->thread, &vp));
+	Lck_Lock(&vwk->mtx);
+	while (!VTAILQ_EMPTY(&vwk->list)) {
+		Lck_Unlock(&vwk->mtx);
+		(void)usleep(100000);
+		Lck_Lock(&vwk->mtx);
+	}
 	AZ(close(vwk->kq));
+	vwk->kq = -1;
+	Lck_Unlock(&vwk->mtx);
+	AZ(pthread_join(vwk->thread, &vp));
 }
 
 /*--------------------------------------------------------------------*/
@@ -210,9 +188,8 @@ const struct waiter_impl waiter_kqueue = {
 	.name =		"kqueue",
 	.init =		vwk_init,
 	.fini =		vwk_fini,
-	.inject =	vwk_inject,
+	.enter =	vwk_enter,
 	.size =		sizeof(struct vwk),
 };
 
 #endif /* defined(HAVE_KQUEUE) */
-#endif
diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c
index a97425a..52a380f 100644
--- a/bin/varnishd/waiter/mgt_waiter.c
+++ b/bin/varnishd/waiter/mgt_waiter.c
@@ -39,10 +39,10 @@
 #include "waiter/waiter_priv.h"
 
 static const struct waiter_impl *const waiter_impls[] = {
-#if 0
     #if defined(HAVE_KQUEUE)
 	&waiter_kqueue,
     #endif
+#if 0
     #if defined(HAVE_EPOLL_CTL)
 	&waiter_epoll,
     #endif



From phk at FreeBSD.org  Thu May 21 17:00:12 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 19:00:12 +0200
Subject: [master] 9500bd7 Also delete the mutex.
Message-ID: <E1YvTpM-0007DF-Ua@project.varnish-software.com>


commit 9500bd79d2c6a154f40729d58fb03ee789591da4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 16:59:52 2015 +0000

    Also delete the mutex.

diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index f5452ac..e7dfdcf 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -180,6 +180,7 @@ vwk_fini(struct waiter *w)
 	vwk->kq = -1;
 	Lck_Unlock(&vwk->mtx);
 	AZ(pthread_join(vwk->thread, &vp));
+	Lck_Delete(&vwk->mtx);
 }
 
 /*--------------------------------------------------------------------*/



From nils.goroll at uplex.de  Thu May 21 17:06:50 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 21 May 2015 19:06:50 +0200
Subject: [master] dc698d7 one fall through comment should be enough for
 everyone
Message-ID: <E1YvTvm-00089R-Ts@project.varnish-software.com>


commit dc698d74e0c9196a82e8998b306cf4a681e22e3f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu May 21 19:06:36 2015 +0200

    one fall through comment should be enough for everyone

diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 22f1a90..7608755 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -119,7 +119,7 @@ RFC2616_Ttl(struct busyobj *bo, double now)
 		 * Do not apply the default ttl, only set a ttl if Cache-Control
 		 * or Expires are present. Uncacheable otherwise.
 		 */
-		expp->ttl = -1.;	/* fall through */
+		expp->ttl = -1.;
 		/* FALL-THROUGH */
 	case 200: /* OK */
 	case 203: /* Non-Authoritative Information */



From nils.goroll at uplex.de  Thu May 21 17:23:30 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 21 May 2015 19:23:30 +0200
Subject: [master] 49712a5 vmods running vmodtool require python,
 so check for it
Message-ID: <E1YvUBv-0005Lu-02@project.varnish-software.com>


commit 49712a5fae5919e6e333d7164b98ef986b149c62
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu May 21 19:21:28 2015 +0200

    vmods running vmodtool require python, so check for it
    
    (even if we exec env python, we should check early if we got one)

diff --git a/varnish.m4 b/varnish.m4
index 8339cb0..b7ad6f9 100644
--- a/varnish.m4
+++ b/varnish.m4
@@ -81,6 +81,10 @@ AC_SUBST([VMOD_DIR])
 
 AC_DEFUN([VARNISH_VMODTOOL],
 [
+AC_CHECK_PROGS(PYTHON, [python3 python3.1 python3.2 python2.7 python2.6 python2.5 python2 python], "no")
+if test "x$PYTHON" = "xno"; then
+  AC_MSG_ERROR([Python is needed to build, please install python.])
+fi
 VARNISH_PKG_GET_VAR([VMODTOOL], [vmodtool])
 AC_SUBST([VMODTOOL])
 ])



From fgsch at lodoss.net  Thu May 21 17:28:16 2015
From: fgsch at lodoss.net (Federico Schwindt)
Date: Thu, 21 May 2015 18:28:16 +0100
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <E1YvUBv-0005Lu-02@project.varnish-software.com>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
Message-ID: <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>

I don't think this is the right way to do it. The varnishapi package should
depend on python.

On Thu, May 21, 2015 at 6:23 PM, Nils Goroll <nils.goroll at uplex.de> wrote:

>
> commit 49712a5fae5919e6e333d7164b98ef986b149c62
> Author: Nils Goroll <nils.goroll at uplex.de>
> Date:   Thu May 21 19:21:28 2015 +0200
>
>     vmods running vmodtool require python, so check for it
>
>     (even if we exec env python, we should check early if we got one)
>
> diff --git a/varnish.m4 b/varnish.m4
> index 8339cb0..b7ad6f9 100644
> --- a/varnish.m4
> +++ b/varnish.m4
> @@ -81,6 +81,10 @@ AC_SUBST([VMOD_DIR])
>
>  AC_DEFUN([VARNISH_VMODTOOL],
>  [
> +AC_CHECK_PROGS(PYTHON, [python3 python3.1 python3.2 python2.7 python2.6
> python2.5 python2 python], "no")
> +if test "x$PYTHON" = "xno"; then
> +  AC_MSG_ERROR([Python is needed to build, please install python.])
> +fi
>  VARNISH_PKG_GET_VAR([VMODTOOL], [vmodtool])
>  AC_SUBST([VMODTOOL])
>  ])
>
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-commit/attachments/20150521/5fef9cad/attachment.html>

From nils.goroll at uplex.de  Thu May 21 18:29:43 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 21 May 2015 20:29:43 +0200
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
Message-ID: <555E2417.4020104@uplex.de>

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

On 21/05/15 19:28, Federico Schwindt wrote:
> I don't think this is the right way to do it. The varnishapi package
> should depend on python.

varnish.m4 is cross platform, package dependencies are not.

So, yes, varnishapi should depend on python, but I still think we should have
this check

- -- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.ccc.de/

http://uplex.de/
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2

iQEcBAEBCAAGBQJVXiQXAAoJEB3Nj1ejhovXa8oH/3tUJiPmUUuHAM/w9IIIyTzx
miAzAfZEqaXQnjjnHHE1AhCtoMjur0EqO1VuSEFuMU1KoSrVlfc3+q8hLs38w15M
SjLVUK74me4VRpOfD6Qk8nqMeD8i32j+y/OLWg+4kOrq2JrN589GHfCA7q9ySUX9
npTBgf5bvrR/3pOKdk2n/P84W5eNpTM2vl4L/q/CjEixjZtEuf6QmGhYHimEZqNo
U7/ruI4vIVUJlcxSJlkIsnavkETDtCBgddZbH/YDkYILrrmt8L4QOBDLFmLQTFz2
VrKPWw1LU3B01eXZfxEciPMCWaeBY6U0QCJp91WmtVrBTY+9bdqELLZMsIJXm3s=
=vkCc
-----END PGP SIGNATURE-----



From nils.goroll at uplex.de  Thu May 21 18:34:56 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 21 May 2015 20:34:56 +0200
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <555E2417.4020104@uplex.de>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
Message-ID: <555E2550.4050404@uplex.de>

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

On 21/05/15 20:29, Nils Goroll wrote:
> So, yes, varnishapi should depend on python

or maybe not so much. Actually only a varnishapi-dev (if this exists, I am not
a package maintainer) should
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2

iQEcBAEBCAAGBQJVXiVPAAoJEB3Nj1ejhovXW48H/iz9cvrsWVjxhvXKA+sXXEIb
iMXtA03pMNXPv4vqYATkPH0ERaDjzuEikk40auYjE3LWOAlOluG4ohPmezF2VXC4
MDsYb+JRBIDj3oEa3uBeAcA2o4ldDujVprXwsODkoo5pwyvTmRKqTED3HND4D+VG
+zSTkZyDL9TSoRMa72tIFBwG5tPMAgRtcGlaDV4El/ZY1qlmJDgGXIK5VadBAvGu
JTxzf5BI8kYtMd+bLwSoMancGVN8K5mci0EpM8x7Xj/DJ1eysk6xUrkQFp7RLNVC
o8lxUKHiFeEvAsnDTsBmEyqWwGytO3JlmV2U/+Tdh4IrJiYXaBcD1t7iLbmoQVQ=
=ii/c
-----END PGP SIGNATURE-----



From phk at phk.freebsd.dk  Thu May 21 19:12:21 2015
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Thu, 21 May 2015 19:12:21 +0000
Subject: [master] dc698d7 one fall through comment should be enough for
 everyone
In-Reply-To: <E1YvTvm-00089R-Ts@project.varnish-software.com>
References: <E1YvTvm-00089R-Ts@project.varnish-software.com>
Message-ID: <13097.1432235541@critter.freebsd.dk>

--------
In message <E1YvTvm-00089R-Ts at project.varnish-software.com>, Nils Goroll writes
:

>    one fall through comment should be enough for everyone

>-		expp->ttl = -1.;	/* fall through */
>+		expp->ttl = -1.;

... but only if it is formatted in a way which lint(1)[1] and decendants
recognize:

	/* FALLTHRU */
	/* FALLTHROUGH */
	/* FALL-THROUGH */

on the line where the "break" would have been.

It's interesting that python solved this by also having the
explicit null-statement "pass".


[1] We should probably use the middle form, I see hints that it is
    the most widely supported.



-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.



From fgsch at lodoss.net  Thu May 21 20:01:54 2015
From: fgsch at lodoss.net (Federico Schwindt)
Date: Thu, 21 May 2015 21:01:54 +0100
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <555E2417.4020104@uplex.de>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
Message-ID: <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>

Precisely. If you are installing from packages python should be a
requirement for the libvarnishapi-dev / varnish-devel package.

If you are compiling from sources (i.e. non-packaged platforms) python is
already required.

On Thu, May 21, 2015 at 7:29 PM, Nils Goroll <nils.goroll at uplex.de> wrote:

> -----BEGIN PGP SIGNED MESSAGE-----
> Hash: SHA256
>
> On 21/05/15 19:28, Federico Schwindt wrote:
> > I don't think this is the right way to do it. The varnishapi package
> > should depend on python.
>
> varnish.m4 is cross platform, package dependencies are not.
>
> So, yes, varnishapi should depend on python, but I still think we should
> have
> this check
>
> - --
>
> ** * * UPLEX - Nils Goroll Systemoptimierung
>
> Scheffelstra?e 32
> 22301 Hamburg
>
> tel +49 40 28805731
> mob +49 170 2723133
> fax +49 40 42949753
>
> xmpp://slink at jabber.ccc.de/
>
> http://uplex.de/
> -----BEGIN PGP SIGNATURE-----
> Version: GnuPG v2
>
> iQEcBAEBCAAGBQJVXiQXAAoJEB3Nj1ejhovXa8oH/3tUJiPmUUuHAM/w9IIIyTzx
> miAzAfZEqaXQnjjnHHE1AhCtoMjur0EqO1VuSEFuMU1KoSrVlfc3+q8hLs38w15M
> SjLVUK74me4VRpOfD6Qk8nqMeD8i32j+y/OLWg+4kOrq2JrN589GHfCA7q9ySUX9
> npTBgf5bvrR/3pOKdk2n/P84W5eNpTM2vl4L/q/CjEixjZtEuf6QmGhYHimEZqNo
> U7/ruI4vIVUJlcxSJlkIsnavkETDtCBgddZbH/YDkYILrrmt8L4QOBDLFmLQTFz2
> VrKPWw1LU3B01eXZfxEciPMCWaeBY6U0QCJp91WmtVrBTY+9bdqELLZMsIJXm3s=
> =vkCc
> -----END PGP SIGNATURE-----
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-commit/attachments/20150521/3eb162ce/attachment.html>

From phk at FreeBSD.org  Thu May 21 20:33:07 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 22:33:07 +0200
Subject: [master] 476589b Stall until waiter is drained.
Message-ID: <E1YvX9P-0007lJ-2s@project.varnish-software.com>


commit 476589bde8f27a11baaa44235aee12799aeca2a2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 20:23:56 2015 +0000

    Stall until waiter is drained.

diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index bb87b17..718a023 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -234,7 +234,10 @@ vwp_init(struct waiter *w)
 	AZ(pthread_create(&vwp->thread, NULL, vwp_main, vwp));
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * It is the callers responsibility to trigger all fd's waited on to
+ * fail somehow.
+ */
 
 static void __match_proto__(waiter_fini_f)
 vwp_fini(struct waiter *w)
@@ -244,6 +247,8 @@ vwp_fini(struct waiter *w)
 
 	CAST_OBJ_NOTNULL(vwp, w->priv, VWP_MAGIC);
 	vp = NULL;
+	while (vwp->hpoll > 1)
+		usleep(100000);
 	// XXX: set write pipe blocking
 	assert(write(vwp->pipes[1], &vp, sizeof vp) == sizeof vp);
 	AZ(pthread_join(vwp->thread, &vp));



From phk at FreeBSD.org  Thu May 21 20:33:07 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 22:33:07 +0200
Subject: [master] 50c9933 Improved shutdown
Message-ID: <E1YvX9P-0007lP-4E@project.varnish-software.com>


commit 50c99336332e9f9590756f2a47d4e222a4ef8ca3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 20:32:59 2015 +0000

    Improved shutdown

diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index e7dfdcf..34aae98 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -55,6 +55,7 @@ struct vwk {
 
 	VTAILQ_HEAD(,waited)	list;
 	struct lock		mtx;
+	int			die;
 };
 
 /*--------------------------------------------------------------------*/
@@ -78,45 +79,44 @@ vwk_thread(void *priv)
 		ts.tv_sec = (time_t)floor(now);
 		ts.tv_nsec = (long)(1e9 * (now - ts.tv_sec));
 		n = kevent(vwk->kq, NULL, 0, ke, NKEV, &ts);
-		if (n < 0 && errno == EBADF)
+		if (n < 0 && vwk->die)
 			break;
+		assert(n >= 0);
 		assert(n <= NKEV);
 		now = VTIM_real();
-		idle = now - *vwk->waiter->tmo;
 		for (kp = ke, j = 0; j < n; j++, kp++) {
 			assert(kp->filter == EVFILT_READ);
 			CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC);
 			Lck_Lock(&vwk->mtx);
 			VTAILQ_REMOVE(&vwk->list, wp, list);
 			Lck_Unlock(&vwk->mtx);
-			if (wp->idle <= idle)
-				vwk->waiter->func(wp, WAITER_TIMEOUT, now);
-			else if (kp->flags & EV_EOF)
+			if (kp->flags & EV_EOF)
 				vwk->waiter->func(wp, WAITER_REMCLOSE, now);
 			else
 				vwk->waiter->func(wp, WAITER_ACTION, now);
 		}
-		if (now - last_idle > .3 * *vwk->waiter->tmo) {
-			last_idle = now;
-			n = 0;
-			Lck_Lock(&vwk->mtx);
-			VTAILQ_FOREACH_SAFE(wp, &vwk->list, list, wp2) {
-				if (wp->idle > idle)
-					continue;
-				EV_SET(ke + n, wp->fd,
-				    EVFILT_READ, EV_DELETE, 0, 0, wp);
-				if (++n == NKEV)
-					break;
-			}
-			if (n > 0)
-				AZ(kevent(vwk->kq, ke, n, NULL, 0, NULL));
-			for (j = 0; j < n; j++) {
-				CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC);
-				VTAILQ_REMOVE(&vwk->list, wp, list);
-				vwk->waiter->func(wp, WAITER_TIMEOUT, now);
-			}
-			Lck_Unlock(&vwk->mtx);
+		idle = now - *vwk->waiter->tmo;
+		if (now - last_idle < .3 * *vwk->waiter->tmo)
+			continue;
+		last_idle = now;
+		n = 0;
+		Lck_Lock(&vwk->mtx);
+		VTAILQ_FOREACH_SAFE(wp, &vwk->list, list, wp2) {
+			if (wp->idle > idle)
+				continue;
+			EV_SET(ke + n, wp->fd,
+			    EVFILT_READ, EV_DELETE, 0, 0, wp);
+			if (++n == NKEV)
+				break;
 		}
+		if (n > 0)
+			AZ(kevent(vwk->kq, ke, n, NULL, 0, NULL));
+		for (j = 0; j < n; j++) {
+			CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC);
+			VTAILQ_REMOVE(&vwk->list, wp, list);
+			vwk->waiter->func(wp, WAITER_TIMEOUT, now);
+		}
+		Lck_Unlock(&vwk->mtx);
 	}
 	return(NULL);
 }
@@ -168,6 +168,7 @@ vwk_fini(struct waiter *w)
 {
 	struct vwk *vwk;
 	void *vp;
+	int i;
 
 	CAST_OBJ_NOTNULL(vwk, w->priv, VWK_MAGIC);
 	Lck_Lock(&vwk->mtx);
@@ -176,8 +177,10 @@ vwk_fini(struct waiter *w)
 		(void)usleep(100000);
 		Lck_Lock(&vwk->mtx);
 	}
-	AZ(close(vwk->kq));
+	vwk->die = 1;
+	i = vwk->kq;
 	vwk->kq = -1;
+	AZ(close(i));
 	Lck_Unlock(&vwk->mtx);
 	AZ(pthread_join(vwk->thread, &vp));
 	Lck_Delete(&vwk->mtx);



From phk at FreeBSD.org  Thu May 21 20:39:11 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 22:39:11 +0200
Subject: [master] d74fd18 Polish
Message-ID: <E1YvXFH-00016E-6P@project.varnish-software.com>


commit d74fd185684012159cedb58880bb71c13b77ea98
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 20:38:10 2015 +0000

    Polish

diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index 718a023..b54d695 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -248,7 +248,7 @@ vwp_fini(struct waiter *w)
 	CAST_OBJ_NOTNULL(vwp, w->priv, VWP_MAGIC);
 	vp = NULL;
 	while (vwp->hpoll > 1)
-		usleep(100000);
+		(void)usleep(100000);
 	// XXX: set write pipe blocking
 	assert(write(vwp->pipes[1], &vp, sizeof vp) == sizeof vp);
 	AZ(pthread_join(vwp->thread, &vp));



From phk at FreeBSD.org  Thu May 21 20:43:49 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 22:43:49 +0200
Subject: [master] 27bab72 Update and reenable epoll(2) waiter to new reality.
Message-ID: <E1YvXJl-0001Hw-Dp@project.varnish-software.com>


commit 27bab7228eab7872c4248f91cae2587178cd7aff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 20:43:07 2015 +0000

    Update and reenable epoll(2) waiter to new reality.

diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index 2e85b87..0452fe9 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -31,7 +31,6 @@
  * write the session pointer to a pipe which the event engine monitors.
  */
 
-#if 0
 #include "config.h"
 
 #if defined(HAVE_EPOLL_CTL)
@@ -56,44 +55,29 @@
 struct vwe {
 	unsigned		magic;
 #define VWE_MAGIC		0x6bd73424
+	int			epfd;
 	struct waiter		*waiter;
-
 	pthread_t		thread;
-	int			epfd;
+	VTAILQ_HEAD(,waited)	list;
+	struct lock		mtx;
+	int			die;
 };
 
 static void
-vwe_inject(const struct waiter *w, struct waited *wp)
-{
-	struct vwe *vwe;
-	struct epoll_event ev;
-
-	CAST_OBJ_NOTNULL(vwe, w->priv, VWE_MAGIC);
-	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
-	assert(wp->fd >= 0);
-
-	ev.data.ptr = wp;
-	ev.events = EPOLLIN | EPOLLRDHUP;
-	AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_ADD, wp->fd, &ev));
-}
-
-static void
 vwe_eev(struct vwe *vwe, const struct epoll_event *ep, double now)
 {
 	struct waited *wp;
 
 	AN(ep->data.ptr);
 	CAST_OBJ_NOTNULL(wp, ep->data.ptr, WAITED_MAGIC);
-	if (wp != vwe->waiter->pipe_w)
-		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL));
 	if (ep->events & EPOLLIN) {
-		Wait_Handle(vwe->waiter, wp, WAITER_ACTION, now);
+		vwe->waiter->func(wp, WAITER_ACTION, now);
 	} else if (ep->events & EPOLLERR) {
-		Wait_Handle(vwe->waiter, wp, WAITER_REMCLOSE, now);
+		vwe->waiter->func(wp, WAITER_REMCLOSE, now);
 	} else if (ep->events & EPOLLHUP) {
-		Wait_Handle(vwe->waiter, wp, WAITER_REMCLOSE, now);
+		vwe->waiter->func(wp, WAITER_REMCLOSE, now);
 	} else if (ep->events & EPOLLRDHUP) {
-		Wait_Handle(vwe->waiter, wp, WAITER_REMCLOSE, now);
+		vwe->waiter->func(wp, WAITER_REMCLOSE, now);
 	}
 }
 
@@ -103,26 +87,77 @@ static void *
 vwe_thread(void *priv)
 {
 	struct epoll_event ev[NEEV], *ep;
-	double now;
+	struct waited *wp, *wp2;
+	double now, idle, last_idle;
 	int i, n;
 	struct vwe *vwe;
+	VTAILQ_HEAD(,waited) tlist;
 
 	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
 
 	THR_SetName("cache-epoll");
 
-	while (!vwe->waiter->dismantle) {
-		n = epoll_wait(vwe->epfd, ev, NEEV, -1);
+	last_idle = 0.0;
+	while (1) {
+		i = floor(.3 * 1e3 * *vwe->waiter->tmo);
+		n = epoll_wait(vwe->epfd, ev, NEEV, i);
+		if (n < 0 && vwe->die)
+			break;
+		assert(n >= 0);
 		now = VTIM_real();
-		for (ep = ev, i = 0; i < n; i++, ep++)
+		for (ep = ev, i = 0; i < n; i++, ep++) {
+			CAST_OBJ_NOTNULL(wp, ep->data.ptr, WAITED_MAGIC);
+			AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL));
+			Lck_Lock(&vwe->mtx);
+			VTAILQ_REMOVE(&vwe->list, wp, list);
+			Lck_Unlock(&vwe->mtx);
 			vwe_eev(vwe, ep, now);
-		Wait_Handle(vwe->waiter, NULL, WAITER_ACTION, now);
+		}
+		idle = now - *vwe->waiter->tmo;
+		if (now - last_idle < .3 * *vwe->waiter->tmo)
+			continue;
+		last_idle = now;
+		VTAILQ_INIT(&tlist);
+		Lck_Lock(&vwe->mtx);
+		VTAILQ_FOREACH_SAFE(wp, &vwe->list, list, wp2) {
+			if (wp->idle > idle)
+				continue;
+			VTAILQ_REMOVE(&vwe->list, wp, list);
+			VTAILQ_INSERT_TAIL(&tlist, wp, list);
+			AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL));
+		}
+		Lck_Unlock(&vwe->mtx);
+		while(1) {
+			wp = VTAILQ_FIRST(&tlist);
+			if (wp == NULL)
+				break;
+			VTAILQ_REMOVE(&tlist, wp, list);
+			vwe->waiter->func(wp, WAITER_TIMEOUT, now);
+		}
 	}
 	return (NULL);
 }
 
 /*--------------------------------------------------------------------*/
 
+static int __match_proto__(waiter_enter_f)
+vwe_enter(void *priv, struct waited *wp)
+{
+	struct vwe *vwe;
+	struct epoll_event ee;
+
+	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
+	ee.events = EPOLLIN | EPOLLRDHUP;
+	ee.data.ptr = wp;
+	Lck_Lock(&vwe->mtx);
+	VTAILQ_INSERT_TAIL(&vwe->list, wp, list);
+	AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_ADD, wp->fd, &ee));
+	Lck_Unlock(&vwe->mtx);
+	return(0);
+}
+
+/*--------------------------------------------------------------------*/
+
 static void __match_proto__(waiter_init_f)
 vwe_init(struct waiter *w)
 {
@@ -135,24 +170,39 @@ vwe_init(struct waiter *w)
 
 	vwe->epfd = epoll_create(1);
 	assert(vwe->epfd >= 0);
-
-	Wait_UsePipe(w);
+	VTAILQ_INIT(&vwe->list);
+	Lck_New(&vwe->mtx, lck_misc);
 
 	AZ(pthread_create(&vwe->thread, NULL, vwe_thread, vwe));
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * It is the callers responsibility to trigger all fd's waited on to
+ * fail somehow.
+ */
 
 static void __match_proto__(waiter_fini_f)
 vwe_fini(struct waiter *w)
 {
 	struct vwe *vwe;
 	void *vp;
+	int i;
 
 	CAST_OBJ_NOTNULL(vwe, w->priv, VWE_MAGIC);
 
+	Lck_Lock(&vwe->mtx);
+	while (!VTAILQ_EMPTY(&vwe->list)) {
+		Lck_Unlock(&vwe->mtx);
+		(void)usleep(100000);
+		Lck_Lock(&vwe->mtx);
+	}
+	vwe->die = 1;
+	i = vwe->epfd;
+	vwe->epfd = -1;
+	AZ(close(i));
+	Lck_Unlock(&vwe->mtx);
 	AZ(pthread_join(vwe->thread, &vp));
-	AZ(close(vwe->epfd));
+	Lck_Delete(&vwe->mtx);
 }
 
 /*--------------------------------------------------------------------*/
@@ -161,9 +211,8 @@ const struct waiter_impl waiter_epoll = {
 	.name =		"epoll",
 	.init =		vwe_init,
 	.fini =		vwe_fini,
-	.inject =	vwe_inject,
+	.enter =	vwe_enter,
 	.size =		sizeof(struct vwe),
 };
 
 #endif /* defined(HAVE_EPOLL_CTL) */
-#endif
diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c
index 52a380f..3f344fd 100644
--- a/bin/varnishd/waiter/mgt_waiter.c
+++ b/bin/varnishd/waiter/mgt_waiter.c
@@ -42,10 +42,10 @@ static const struct waiter_impl *const waiter_impls[] = {
     #if defined(HAVE_KQUEUE)
 	&waiter_kqueue,
     #endif
-#if 0
     #if defined(HAVE_EPOLL_CTL)
 	&waiter_epoll,
     #endif
+#if 0
     #if defined(HAVE_PORT_CREATE)
 	&waiter_ports,
     #endif



From phk at FreeBSD.org  Thu May 21 21:39:58 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 21 May 2015 23:39:58 +0200
Subject: [master] 12ace86 Stabilize this test a bit
Message-ID: <E1YvYC6-0007ii-5m@project.varnish-software.com>


commit 12ace86f88c18a1ec85b18578e6f36ae2d95d501
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu May 21 21:39:43 2015 +0000

    Stabilize this test a bit

diff --git a/bin/varnishtest/tests/r00979.vtc b/bin/varnishtest/tests/r00979.vtc
index 46d7fa9..80ae0c2 100644
--- a/bin/varnishtest/tests/r00979.vtc
+++ b/bin/varnishtest/tests/r00979.vtc
@@ -2,11 +2,12 @@ varnishtest "r00979.vtc Test restart when do_stream in vcl_deliver"
 
 server s1 {
 	rxreq
-	txresp -status 200 -gzipbody "1"
+	txresp -hdr "Connection: close" -gzipbody "1"
 
+	expect_close
 	accept
 	rxreq
-	txresp -status 200 -body "11"
+	txresp -body "11"
 } -start
 
 varnish v1 -vcl+backend {



From slink at schokola.de  Thu May 21 22:03:30 2015
From: slink at schokola.de (Nils Goroll)
Date: Fri, 22 May 2015 00:03:30 +0200
Subject: [master] dc698d7 one fall through comment should be enough for
 everyone
In-Reply-To: <13097.1432235541@critter.freebsd.dk>
References: <E1YvTvm-00089R-Ts@project.varnish-software.com>
 <13097.1432235541@critter.freebsd.dk>
Message-ID: <555E5632.1040200@schokola.de>



On 21/05/15 21:12, Poul-Henning Kamp wrote:
> --------
> In message <E1YvTvm-00089R-Ts at project.varnish-software.com>, Nils Goroll writes
> :
> 
>>    one fall through comment should be enough for everyone
> 
>> -		expp->ttl = -1.;	/* fall through */
>> +		expp->ttl = -1.;
> 
> ... but only if it is formatted in a way which lint(1)[1] and decendants
> recognize:
> 
> 	/* FALLTHRU */
> 	/* FALLTHROUGH */
> 	/* FALL-THROUGH */
> 
> on the line where the "break" would have been.

And you sure have noticed that it was this form I have kept. Thanks for digging
out the full details about the best form!




From slink at schokola.de  Thu May 21 22:05:04 2015
From: slink at schokola.de (Nils Goroll)
Date: Fri, 22 May 2015 00:05:04 +0200
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
 <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>
Message-ID: <555E5690.8090706@schokola.de>

On 21/05/15 22:01, Federico Schwindt wrote:
> 
> If you are compiling from sources (i.e. non-packaged platforms) python is
> already required.

Sorry, no. If I install a varnish-cache binary tar on a system and want to build
a vmod from sources, there is no other dependency checking than that from the vmod.

Nils



From phk at FreeBSD.org  Fri May 22 06:36:03 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 08:36:03 +0200
Subject: [master] da26134 Stabilize this test
Message-ID: <E1YvgYt-0001jF-9L@project.varnish-software.com>


commit da2613492d02c1c26a97c3808d23dedd8e0eadde
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 06:35:53 2015 +0000

    Stabilize this test

diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc
index 154c15c..458718e 100644
--- a/bin/varnishtest/tests/c00055.vtc
+++ b/bin/varnishtest/tests/c00055.vtc
@@ -3,11 +3,11 @@ varnishtest "test caching of req.body"
 server s1 {
 	rxreq
 	expect req.bodylen == 3
-	txresp -status 200 -hdr "Foo: BAR" -body "1234"
+	txresp -hdr "Connection: close" -hdr "Foo: BAR" -body "1234"
 	accept
 	rxreq
 	expect req.bodylen == 3
-	txresp -status 200 -hdr "Foo: Foo" -body "56"
+	txresp -hdr "Foo: Foo" -body "56"
 } -start
 
 varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend {



From phk at FreeBSD.org  Fri May 22 09:58:00 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 11:58:00 +0200
Subject: [master] 71dfce8 Only explicitly show param default if it not set to
 that default
Message-ID: <E1YvjiK-00026e-IQ@project.varnish-software.com>


commit 71dfce8866947639f38953b834a014c520a82725
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 09:21:33 2015 +0000

    Only explicitly show param default if it not set to that default

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 4f6e5b8..bead46b 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -245,8 +245,9 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 			VCLI_Out(cli, " (default)");
 		VCLI_Out(cli, "\n");
 		if (lfmt) {
-			VCLI_Out(cli, "%-*sDefault is: %s\n",
-			    margin1, "", pp->def);
+			if (pp->def != NULL && strcmp(pp->def, VSB_data(vsb)))
+				VCLI_Out(cli, "%-*sDefault is: %s\n",
+				    margin1, "", pp->def);
 			if (pp->min != NULL)
 				VCLI_Out(cli, "%-*sMinimum is: %s\n",
 				    margin1, "", pp->min);



From phk at FreeBSD.org  Fri May 22 09:58:00 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 11:58:00 +0200
Subject: [master] 0cba6f8 Convert the parameter array to a list of parameters.
Message-ID: <E1YvjiK-00026k-JV@project.varnish-software.com>


commit 0cba6f823757ed28df8bd6e695384e40ef0ea2da
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 09:56:48 2015 +0000

    Convert the parameter array to a list of parameters.

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index bead46b..1fd3f33 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -46,9 +46,16 @@
 
 #include "mgt_cli.h"
 
+struct plist {
+	unsigned			magic;
+#define PLIST_MAGIC			0xbfc3ea16
+	VTAILQ_ENTRY(plist)		list;
+	struct parspec			*spec;
+};
+
+static VTAILQ_HEAD(, plist)		phead = VTAILQ_HEAD_INITIALIZER(phead);
+
 struct params mgt_param;
-static int nparspec;
-static struct parspec ** parspecs;
 static const int margin1 = 8;
 static int margin2 = 0;
 static const int wrap_at = 72;
@@ -94,21 +101,42 @@ static const char ONLY_ROOT_TEXT[] =
 	"\n\n"
 	"NB: This parameter only works if varnishd is run as root.";
 
-
 /*--------------------------------------------------------------------*/
 
 static struct parspec *
 mcf_findpar(const char *name)
 {
-	int i;
+	struct plist *pl;
 
 	AN(name);
-	for (i = 0; i < nparspec; i++)
-		if (!strcmp(parspecs[i]->name, name))
-			return (parspecs[i]);
+	VTAILQ_FOREACH(pl, &phead, list)
+		if (!strcmp(pl->spec->name, name))
+			return (pl->spec);
 	return (NULL);
 }
 
+static void
+mcf_addpar(struct parspec *ps)
+{
+	struct plist *pl, *pl2;
+	int i;
+
+	ALLOC_OBJ(pl, PLIST_MAGIC);
+	AN(pl);
+	pl->spec = ps;
+	VTAILQ_FOREACH(pl2, &phead, list) {
+		i = strcmp(pl2->spec->name, pl->spec->name);
+		if (i == 0) {
+			fprintf(stderr, "Duplicate param: %s\n", ps->name);
+			exit(4);
+		} else if (i > 0) {
+			VTAILQ_INSERT_BEFORE(pl2, pl, list);
+			return;
+		}
+	}
+	VTAILQ_INSERT_TAIL(&phead, pl, list);
+}
+
 /*--------------------------------------------------------------------
  * Wrap the text nicely.
  * Lines are allowed to contain to TABS and we render that as a table
@@ -205,7 +233,8 @@ mcf_wrap(struct cli *cli, const char *text)
 void
 mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 {
-	int i, n;
+	int n;
+	struct plist *pl;
 	const struct parspec *pp;
 	int lfmt = 0, chg = 0;
 	struct vsb *vsb;
@@ -219,8 +248,8 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 		lfmt = 1;
 
 	n = 0;
-	for (i = 0; i < nparspec; i++) {
-		pp = parspecs[i];
+	VTAILQ_FOREACH(pl, &phead, list) {
+		pp = pl->spec;
 		if (lfmt && strcmp(pp->name, av[2]) && strcmp("-l", av[2]))
 			continue;
 		n++;
@@ -364,22 +393,12 @@ mcf_param_set(struct cli *cli, const char * const *av, void *priv)
  * Add a group of parameters to the global set and sort by name.
  */
 
-static int
-mcf_parspec_cmp(const void *a, const void *b)
-{
-	struct parspec * const * pa = a;
-	struct parspec * const * pb = b;
-	return (strcmp((*pa)->name, (*pb)->name));
-}
-
 void
 MCF_AddParams(struct parspec *ps)
 {
 	struct parspec *pp;
 	const char *s;
-	int n;
 
-	n = 0;
 	for (pp = ps; pp->name != NULL; pp++) {
 		AN(pp->func);
 		s = strchr(pp->descr, '\0');
@@ -388,23 +407,12 @@ MCF_AddParams(struct parspec *ps)
 			    "Param->descr has trailing space: %s\n", pp->name);
 			exit(4);
 		}
-		if (mcf_findpar(pp->name) != NULL) {
-			fprintf(stderr, "Duplicate param: %s\n", pp->name);
-			exit(4);
-		}
+		mcf_addpar(pp);
 		if (strlen(pp->name) + 1 > margin2)
 			margin2 = strlen(pp->name) + 1;
-		n++;
 	}
-	parspecs = realloc(parspecs, (1L + nparspec + n) * sizeof *parspecs);
-	XXXAN(parspecs);
-	for (pp = ps; pp->name != NULL; pp++)
-		parspecs[nparspec++] = pp;
-	parspecs[nparspec] = NULL;
-	qsort (parspecs, nparspec, sizeof parspecs[0], mcf_parspec_cmp);
 }
 
-
 /*--------------------------------------------------------------------
  * Wash a min/max/default value
  */
@@ -443,14 +451,14 @@ mcf_wash_param(struct cli *cli, const struct parspec *pp, const char **val,
 void
 MCF_InitParams(struct cli *cli)
 {
+	struct plist *pl;
 	struct parspec *pp;
-	int i;
 	struct vsb *vsb;
 
 	vsb = VSB_new_auto();
 	AN(vsb);
-	for (i = 0; i < nparspec; i++) {
-		pp = parspecs[i];
+	VTAILQ_FOREACH(pl, &phead, list) {
+		pp = pl->spec;
 
 		if (pp->min != NULL)
 			mcf_wash_param(cli, pp, &pp->min, "minimum", vsb);
@@ -515,14 +523,15 @@ MCF_SetMaximum(const char *param, const char *new_max)
 void
 MCF_DumpRstParam(void)
 {
+	struct plist *pl;
 	const struct parspec *pp;
 	const char *p, *q, *t1, *t2;
-	int i, j;
+	int j;
 
 	printf("\n.. The following is the autogenerated "
 	    "output from varnishd -x dumprstparam\n\n");
-	for (i = 0; i < nparspec; i++) {
-		pp = parspecs[i];
+	VTAILQ_FOREACH(pl, &phead, list) {
+		pp = pl->spec;
 		printf(".. _ref_param_%s:\n\n", pp->name);
 		printf("%s\n", pp->name);
 		for (j = 0; j < strlen(pp->name); j++)



From phk at FreeBSD.org  Fri May 22 10:51:39 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 12:51:39 +0200
Subject: [master] 3327577 Only look for accept-filters if platform supports
 them.
Message-ID: <E1YvkYF-0006fD-Om@project.varnish-software.com>


commit 3327577283bc3da96278048f57ece99948c444a3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 10:49:59 2015 +0000

    Only look for accept-filters if platform supports them.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 0fb6349..210737e 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -489,6 +489,7 @@ vca_acct(void *arg)
 		assert (ls->sock > 0);		// We know where stdin is
 		AZ(listen(ls->sock, cache_param->listen_depth));
 		vca_tcp_opt_set(ls->sock, 1);
+#ifdef HAVE_ACCEPT_FILTERS
 		if (cache_param->accept_filter) {
 			i = VTCP_filter_http(ls->sock);
 			if (i)
@@ -496,6 +497,7 @@ vca_acct(void *arg)
 				    "Kernel filtering: sock=%d, ret=%d %s",
 				    ls->sock, i, strerror(errno));
 		}
+#endif /* HAVE_ACCEPT_FILTERS */
 	}
 
 	need_test = 1;



From phk at FreeBSD.org  Fri May 22 10:51:39 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 12:51:39 +0200
Subject: [master] 39e0bc5 Start tabelizing the trivial parameters.
Message-ID: <E1YvkYF-0006fJ-Pz@project.varnish-software.com>


commit 39e0bc53b51288d7999e67bc0f15b9b5aeb53d3c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 10:50:22 2015 +0000

    Start tabelizing the trivial parameters.
    
    We will not be able to put the more complex and compound parameters
    into this table, but it will help keep types in struct param and
    tweak_functions in sync.
    
    Feel free to help get through the list.

diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index 72b74d6..a20fe47 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -57,6 +57,16 @@ struct poolparam {
 
 struct params {
 
+#define	ptyp_bool	unsigned
+#define	ptyp_double	double
+#define	ptyp_timeout	double
+#define PARAM(nm, ty, mi, ma, de, un, fl, st, lt, fn) ptyp_##ty nm;
+#include <tbl/params.h>
+#undef PARAM
+#undef ptyp_bool
+#undef ptyp_double
+#undef ptyp_timeout
+
 	/* Unprivileged user / group */
 	char			*user;
 	uid_t			uid;
@@ -128,8 +138,6 @@ struct params {
 	ssize_t			fetch_maxchunksize;
 	unsigned		nuke_limit;
 
-	unsigned		accept_filter;
-
 	/* Listen depth */
 	unsigned		listen_depth;
 
@@ -169,11 +177,6 @@ struct params {
 	/* Acceptable clockskew with backends */
 	unsigned		clock_skew;
 
-	/* Acceptor pacer parameters */
-	double			acceptor_sleep_max;
-	double			acceptor_sleep_incr;
-	double			acceptor_sleep_decay;
-
 	/* Get rid of duplicate bans */
 	unsigned		ban_dups;
 
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index ae69bee..e59d38b 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -45,6 +45,11 @@
 	"\tmax_age\tmax age of free element."
 
 struct parspec mgt_parspec[] = {
+#define PARAM(nm, ty, mi, ma, de, un, fl, st, lt, fn)		\
+	{ #nm, tweak_##ty, &mgt_param.nm, mi, ma, st, fl, de, un },
+#include "tbl/params.h"
+#undef PARAM
+
 	{ "default_ttl", tweak_timeout, &mgt_param.default_ttl,
 		"0", NULL,
 		"The TTL assigned to objects if neither the backend nor "
@@ -231,13 +236,6 @@ struct parspec mgt_parspec[] = {
 		"fragmentation.",
 		EXPERIMENTAL,
 		"256m", "bytes" },
-#ifdef HAVE_ACCEPT_FILTERS
-	{ "accept_filter", tweak_bool, &mgt_param.accept_filter,
-		NULL, NULL,
-		"Enable kernel accept-filters, (if available in the kernel).",
-		MUST_RESTART,
-		"on", "bool" },
-#endif
 	{ "listen_depth", tweak_uint, &mgt_param.listen_depth,
 		"0", NULL,
 		"Listen queue depth.",
@@ -338,33 +336,6 @@ struct parspec mgt_parspec[] = {
 		"and backend request. This parameter does not apply to pipe.",
 		0,
 		"60", "seconds" },
-	{ "acceptor_sleep_max", tweak_timeout,
-		&mgt_param.acceptor_sleep_max,
-		"0", "10",
-		"If we run out of resources, such as file descriptors or "
-		"worker threads, the acceptor will sleep between accepts.\n"
-		"This parameter limits how long it can sleep between "
-		"attempts to accept new connections.",
-		EXPERIMENTAL,
-		"0.050", "seconds" },
-	{ "acceptor_sleep_incr", tweak_timeout,
-		&mgt_param.acceptor_sleep_incr,
-		"0", "1",
-		"If we run out of resources, such as file descriptors or "
-		"worker threads, the acceptor will sleep between accepts.\n"
-		"This parameter control how much longer we sleep, each time "
-		"we fail to accept a new connection.",
-		EXPERIMENTAL,
-		"0.001", "seconds" },
-	{ "acceptor_sleep_decay", tweak_double,
-		&mgt_param.acceptor_sleep_decay,
-		"0", "1",
-		"If we run out of resources, such as file descriptors or "
-		"worker threads, the acceptor will sleep between accepts.\n"
-		"This parameter (multiplicatively) reduce the sleep duration "
-		"for each successful accept. (ie: 0.9 = reduce by 10%)",
-		EXPERIMENTAL,
-		"0.900", "" },
 	{ "clock_skew", tweak_uint, &mgt_param.clock_skew,
 		"0", NULL,
 		"How much clockskew we are willing to accept between the "
diff --git a/include/Makefile.am b/include/Makefile.am
index 1f116b6..2d4593e 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -14,6 +14,7 @@ nobase_pkginclude_HEADERS = \
 	tbl/http_response.h \
 	tbl/locks.h \
 	tbl/obj_attr.h \
+	tbl/params.h \
 	tbl/req_body.h \
 	tbl/req_flags.h \
 	tbl/sess_attr.h \
diff --git a/include/tbl/params.h b/include/tbl/params.h
new file mode 100644
index 0000000..78ee42b
--- /dev/null
+++ b/include/tbl/params.h
@@ -0,0 +1,1549 @@
+/*-
+ * Copyright (c) 2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * PARAM(nm, ty, mi, ma, de, un, fl, st, lt, fn)
+ */
+
+/*lint -save -e525 -e539 */
+
+#ifdef HAVE_ACCEPT_FILTERS
+PARAM(
+	/* name */	accept_filter,
+	/* typ */	bool,
+	/* min */	NULL,
+	/* max */	NULL,
+	/* default */	"on",
+	/* units */	"bool",
+	/* flags */	MUST_RESTART,
+	/* s-text */
+	"Enable kernel accept-filters, (if available in the kernel).",
+	/* l-text */	NULL,
+	/* func */	NULL
+)
+#endif /* HAVE_ACCEPT_FILTERS */
+
+PARAM(
+	/* name */	acceptor_sleep_decay,
+	/* typ */	double,
+	/* min */	"0",
+	/* max */	"1",
+	/* default */	"0.9",
+	/* units */	NULL,
+	/* flags */	EXPERIMENTAL,
+	/* s-text */
+	"If we run out of resources, such as file descriptors or worker "
+	"threads, the acceptor will sleep between accepts.\n"
+	"This parameter (multiplicatively) reduce the sleep duration for "
+	"each successful accept. (ie: 0.9 = reduce by 10%)",
+	/* l-text */	"",
+	/* func */	NULL
+)
+
+PARAM(
+	/* name */	acceptor_sleep_incr,
+	/* typ */	timeout,
+	/* min */	"0",
+	/* max */	"1",
+	/* default */	"0",
+	/* units */	"seconds",
+	/* flags */	EXPERIMENTAL,
+	/* s-text */
+	"If we run out of resources, such as file descriptors or worker "
+	"threads, the acceptor will sleep between accepts.\n"
+	"This parameter control how much longer we sleep, each time we "
+	"fail to accept a new connection.",
+	/* l-text */	"",
+	/* func */	NULL
+)
+
+PARAM(
+	/* name */	acceptor_sleep_max,
+	/* typ */	timeout,
+	/* min */	"0",
+	/* max */	"10",
+	/* default */	"0.05",
+	/* units */	"seconds",
+	/* flags */	EXPERIMENTAL,
+	/* s-text */
+	"If we run out of resources, such as file descriptors or worker "
+	"threads, the acceptor will sleep between accepts.\n"
+	"This parameter limits how long it can sleep between attempts to "
+	"accept new connections.",
+	/* l-text */	"",
+	/* func */	NULL
+)
+
+/**********************************************************************/
+#if 0 /* NOT YET */
+PARAM(
+	/* name */	auto_restart,
+	/* tweak */	tweak_bool,
+	/* var */	auto_restart,
+	/* min */	none,
+	/* max */	none,
+	/* default */	on,
+	/* units */	bool,
+	/* flags */	00,
+	/* s-text */
+	"Restart child process automatically if it dies.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	ban_dups,
+	/* tweak */	tweak_bool,
+	/* var */	ban_dups,
+	/* min */	none,
+	/* max */	none,
+	/* default */	on,
+	/* units */	bool,
+	/* flags */	00,
+	/* s-text */
+	"Eliminate older identical bans when new bans are created.  This "
+	"test is CPU intensive and scales with the number and complexity "
+	"of active (non-Gone) bans.  If identical bans are frequent, the "
+	"amount of CPU needed to actually test  the bans will be similarly "
+	"reduced.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	ban_lurker_age,
+	/* tweak */	tweak_timeout,
+	/* var */	ban_lurker_age,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	60.000,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"The ban lurker does not process bans until they are this old.  "
+	"Right when a ban is added, the most frequently hit objects will "
+	"get tested against it as part of object lookup.  This parameter "
+	"prevents the ban-lurker from kicking in, until the rush is over.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	ban_lurker_batch,
+	/* tweak */	tweak_uint,
+	/* var */	ban_lurker_batch,
+	/* min */	1,
+	/* max */	none,
+	/* default */	1000,
+	/* units */	,
+	/* flags */	00,
+	/* s-text */
+	"How many objects the ban lurker examines before taking a "
+	"ban_lurker_sleep.  Use this to pace the ban lurker so it does not "
+	"eat too much CPU.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	ban_lurker_sleep,
+	/* tweak */	tweak_timeout,
+	/* var */	ban_lurker_sleep,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	0.010,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"The ban lurker thread sleeps between work batches, in order to "
+	"not monopolize CPU power.  When nothing is done, it sleeps a "
+	"fraction of a second before looking for new work to do.\n"
+	"A value of zero disables the ban lurker.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	between_bytes_timeout,
+	/* tweak */	tweak_timeout,
+	/* var */	between_bytes_timeout,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	60.000,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"Default timeout between bytes when receiving data from backend. "
+	"We only wait for this many seconds between bytes before giving "
+	"up. A value of 0 means it will never time out. VCL can override "
+	"this default value for each backend request and backend request. "
+	"This parameter does not apply to pipe.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	busyobj_worker_cache,
+	/* tweak */	tweak_bool,
+	/* var */	busyobj_worker_cache,
+	/* min */	none,
+	/* max */	none,
+	/* default */	off,
+	/* units */	bool,
+	/* flags */	00,
+	/* s-text */
+	"Cache free busyobj per worker thread. Disable this if you have "
+	"very high hitrates and want to save the memory of one busyobj per "
+	"worker thread.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+#if 0
+PARAM(
+	/* name */	cc_command,
+	/* tweak */	tweak_string,
+	/* var */	cc_command,
+	/* min */	none,
+	/* max */	none,
+	/* default */	"exec clang -std=gnu99 -g -O2 -Wall -Werror -Wno-error=unused-result  \t-Werror \t-Wall \t-Wno-format-y2k \t-W \t-Wstrict-prototypes \t-Wmissing-prototypes \t-Wpointer-arith \t-Wreturn-type \t-Wcast-qual \t-Wwrite-strings \t-Wswitch \t-Wshadow \t-Wunused-parameter \t-Wcast-align \t-Wchar-subscripts \t-Wnested-externs \t-Wextra \t-Wno-sign-compare  -fstack-protector -Wno-pointer-sign -Wno-address -Wno-missing-field-initializers -D_THREAD_SAFE -pthread -fpic -shared -Wl,-x -o %o %s",
+	/* units */	(null),
+	/* flags */	0| MUST_RELOAD,
+	/* s-text */
+	"Command used for compiling the C source code to a dlopen(3) "
+	"loadable object.  Any occurrence of %s in the string will be "
+	"replaced with the source file name, and %o will be replaced with "
+	"the output file name.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	cli_buffer,
+	/* tweak */	tweak_bytes_u,
+	/* var */	cli_buffer,
+	/* min */	4k,
+	/* max */	none,
+	/* default */	8k,
+	/* units */	bytes,
+	/* flags */	00,
+	/* s-text */
+	"Size of buffer for CLI command input.\n"
+	"You may need to increase this if you have big VCL files and use "
+	"the vcl.inline CLI command.\n"
+	"NB: Must be specified with -p to have effect.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	cli_limit,
+	/* tweak */	tweak_bytes_u,
+	/* var */	cli_limit,
+	/* min */	128b,
+	/* max */	99999999b,
+	/* default */	48k,
+	/* units */	bytes,
+	/* flags */	00,
+	/* s-text */
+	"Maximum size of CLI response.  If the response exceeds this "
+	"limit, the response code will be 201 instead of 200 and the last "
+	"line will indicate the truncation.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	cli_timeout,
+	/* tweak */	tweak_timeout,
+	/* var */	cli_timeout,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	60.000,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"Timeout for the childs replies to CLI requests from the "
+	"mgt_param.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	clock_skew,
+	/* tweak */	tweak_uint,
+	/* var */	clock_skew,
+	/* min */	0,
+	/* max */	none,
+	/* default */	10,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"How much clockskew we are willing to accept between the backend "
+	"and our own clock.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	connect_timeout,
+	/* tweak */	tweak_timeout,
+	/* var */	connect_timeout,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	3.500,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"Default connection timeout for backend connections. We only try "
+	"to connect to the backend for this many seconds before giving up. "
+	"VCL can override this default value for each backend and backend "
+	"request.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	critbit_cooloff,
+	/* tweak */	tweak_timeout,
+	/* var */	critbit_cooloff,
+	/* min */	60.000,
+	/* max */	254.000,
+	/* default */	180.000,
+	/* units */	seconds,
+	/* flags */	0| WIZARD,
+	/* s-text */
+	"How long the critbit hasher keeps deleted objheads on the cooloff "
+	"list.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	debug,
+	/* tweak */	tweak_mask,
+	/* var */	debug,
+	/* min */	none,
+	/* max */	none,
+	/* default */	none,
+	/* units */	,
+	/* flags */	0,
+	/* s-text */
+	"Enable/Disable various kinds of debugging.\n"
+	"	none	Disable all debugging\n"
+	"\n"
+	"Use +/- prefix to set/reset individual bits:\n"
+	"	req_state	VSL Request state engine\n"
+	"	workspace	VSL Workspace operations\n"
+	"	waiter	VSL Waiter internals\n"
+	"	waitinglist	VSL Waitinglist events\n"
+	"	syncvsl	Make VSL synchronous\n"
+	"	hashedge	Edge cases in Hash\n"
+	"	vclrel	Rapid VCL release\n"
+	"	lurker	VSL Ban lurker\n"
+	"	esi_chop	Chop ESI fetch to bits\n"
+	"	flush_head	Flush after http1 head\n"
+	"	vtc_mode	Varnishtest Mode\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	default_grace,
+	/* tweak */	tweak_timeout,
+	/* var */	default_grace,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	10.000,
+	/* units */	seconds,
+	/* flags */	0,
+	/* s-text */
+	"Default grace period.  We will deliver an object this long after "
+	"it has expired, provided another thread is attempting to get a "
+	"new copy.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	default_keep,
+	/* tweak */	tweak_timeout,
+	/* var */	default_keep,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	0.000,
+	/* units */	seconds,
+	/* flags */	0,
+	/* s-text */
+	"Default keep period.  We will keep a useless object around this "
+	"long, making it available for conditional backend fetches.  That "
+	"means that the object will be removed from the cache at the end "
+	"of ttl+grace+keep.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	default_ttl,
+	/* tweak */	tweak_timeout,
+	/* var */	default_ttl,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	120.000,
+	/* units */	seconds,
+	/* flags */	0,
+	/* s-text */
+	"The TTL assigned to objects if neither the backend nor the VCL "
+	"code assigns one.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	feature,
+	/* tweak */	tweak_mask
+	/* var */	feature,
+	/* min */	none,
+	/* max */	none,
+	/* default */	none,
+	/* units */	,
+	/* flags */	00,
+	/* s-text */
+	"Enable/Disable various minor features.\n"
+	"	none	Disable all features.\n"
+	"\n"
+	"Use +/- prefix to enable/disable individual feature:\n"
+	"	short_panic	Short panic message.\n"
+	"	wait_silo	Wait for persistent silo.\n"
+	"	no_coredump	No coredumps.\n"
+	"	esi_ignore_https	Treat HTTPS as HTTP in ESI:includes\n"
+	"	esi_disable_xml_check	Don't check of body looks like XML\n"
+	"	esi_ignore_other_elements	Ignore non-esi XML-elements\n"
+	"	esi_remove_bom	Remove UTF-8 BOM\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	fetch_chunksize,
+	/* tweak */	tweak_bytes,
+	/* var */	fetch_chunksize,
+	/* min */	4k,
+	/* max */	none,
+	/* default */	16k,
+	/* units */	bytes,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"The default chunksize used by fetcher. This should be bigger than "
+	"the majority of objects with short TTLs.\n"
+	"Internal limits in the storage_file module makes increases above "
+	"128kb a dubious idea.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	fetch_maxchunksize,
+	/* tweak */	tweak_bytes,
+	/* var */	fetch_maxchunksize,
+	/* min */	64k,
+	/* max */	none,
+	/* default */	0.25G,
+	/* units */	bytes,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"The maximum chunksize we attempt to allocate from storage. Making "
+	"this too large may cause delays and storage fragmentation.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	first_byte_timeout,
+	/* tweak */	tweak_timeout,
+	/* var */	first_byte_timeout,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	60.000,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"Default timeout for receiving first byte from backend. We only "
+	"wait for this many seconds for the first byte before giving up. A "
+	"value of 0 means it will never time out. VCL can override this "
+	"default value for each backend and backend request. This "
+	"parameter does not apply to pipe.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	gzip_buffer,
+	/* tweak */	tweak_bytes_u,
+	/* var */	gzip_buffer,
+	/* min */	2k,
+	/* max */	none,
+	/* default */	32k,
+	/* units */	bytes,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"Size of malloc buffer used for gzip processing.\n"
+	"These buffers are used for in-transit data, for instance "
+	"gunzip'ed data being sent to a client.Making this space to small "
+	"results in more overhead, writes to sockets etc, making it too "
+	"big is probably just a waste of memory.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	gzip_level,
+	/* tweak */	tweak_uint,
+	/* var */	gzip_level,
+	/* min */	0,
+	/* max */	9,
+	/* default */	6,
+	/* units */	,
+	/* flags */	00,
+	/* s-text */
+	"Gzip compression level: 0=debug, 1=fast, 9=best\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	gzip_memlevel,
+	/* tweak */	tweak_uint,
+	/* var */	gzip_memlevel,
+	/* min */	1,
+	/* max */	9,
+	/* default */	8,
+	/* units */	,
+	/* flags */	00,
+	/* s-text */
+	"Gzip memory level 1=slow/least, 9=fast/most compression.\n"
+	"Memory impact is 1=1k, 2=2k, ... 9=256k.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	http_gzip_support,
+	/* tweak */	tweak_bool,
+	/* var */	http_gzip_support,
+	/* min */	none,
+	/* max */	none,
+	/* default */	on,
+	/* units */	bool,
+	/* flags */	00,
+	/* s-text */
+	"Enable gzip support. When enabled Varnish request compressed "
+	"objects from the backend and store them compressed. If a client "
+	"does not support gzip encoding Varnish will uncompress compressed "
+	"objects on demand. Varnish will also rewrite the Accept-Encoding "
+	"header of clients indicating support for gzip to:\n"
+	"  Accept-Encoding: gzip\n"
+	"\n"
+	"Clients that do not support gzip will have their Accept-Encoding "
+	"header removed. For more information on how gzip is implemented "
+	"please see the chapter on gzip in the Varnish reference.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	http_max_hdr,
+	/* tweak */	tweak_uint,
+	/* var */	http_max_hdr,
+	/* min */	32,
+	/* max */	65535,
+	/* default */	64,
+	/* units */	header lines,
+	/* flags */	00,
+	/* s-text */
+	"Maximum number of HTTP header lines we allow in "
+	"{req|resp|bereq|beresp}.http (obj.http is autosized to the exact "
+	"number of headers).\n"
+	"Cheap, ~20 bytes, in terms of workspace memory.\n"
+	"Note that the first line occupies five header lines.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	http_range_support,
+	/* tweak */	tweak_bool,
+	/* var */	http_range_support,
+	/* min */	none,
+	/* max */	none,
+	/* default */	on,
+	/* units */	bool,
+	/* flags */	00,
+	/* s-text */
+	"Enable support for HTTP Range headers.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	http_req_hdr_len,
+	/* tweak */	tweak_bytes_u,
+	/* var */	http_req_hdr_len,
+	/* min */	40b,
+	/* max */	none,
+	/* default */	8k,
+	/* units */	bytes,
+	/* flags */	00,
+	/* s-text */
+	"Maximum length of any HTTP client request header we will allow.  "
+	"The limit is inclusive its continuation lines.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	http_req_size,
+	/* tweak */	tweak_bytes_u,
+	/* var */	http_req_size,
+	/* min */	0.25k,
+	/* max */	none,
+	/* default */	32k,
+	/* units */	bytes,
+	/* flags */	00,
+	/* s-text */
+	"Maximum number of bytes of HTTP client request we will deal with. "
+	" This is a limit on all bytes up to the double blank line which "
+	"ends the HTTP request.\n"
+	"The memory for the request is allocated from the client workspace "
+	"(param: workspace_client) and this parameter limits how much of "
+	"that the request is allowed to take up.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	http_resp_hdr_len,
+	/* tweak */	tweak_bytes_u,
+	/* var */	http_resp_hdr_len,
+	/* min */	40b,
+	/* max */	none,
+	/* default */	8k,
+	/* units */	bytes,
+	/* flags */	00,
+	/* s-text */
+	"Maximum length of any HTTP backend response header we will allow. "
+	" The limit is inclusive its continuation lines.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	http_resp_size,
+	/* tweak */	tweak_bytes_u,
+	/* var */	http_resp_size,
+	/* min */	0.25k,
+	/* max */	none,
+	/* default */	32k,
+	/* units */	bytes,
+	/* flags */	00,
+	/* s-text */
+	"Maximum number of bytes of HTTP backend response we will deal "
+	"with.  This is a limit on all bytes up to the double blank line "
+	"which ends the HTTP request.\n"
+	"The memory for the request is allocated from the worker workspace "
+	"(param: thread_pool_workspace) and this parameter limits how much "
+	"of that the request is allowed to take up.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	idle_send_timeout,
+	/* tweak */	tweak_timeout,
+	/* var */	idle_send_timeout,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	60.000,
+	/* units */	seconds,
+	/* flags */	0| DELAYED_EFFECT,
+	/* s-text */
+	"Time to wait with no data sent. If no data has been transmitted "
+	"in this many\n"
+	"seconds the session is closed.\n"
+	"See setsockopt(2) under SO_SNDTIMEO for more information.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	listen_depth,
+	/* tweak */	tweak_uint,
+	/* var */	listen_depth,
+	/* min */	0,
+	/* max */	none,
+	/* default */	1024,
+	/* units */	connections,
+	/* flags */	0| MUST_RESTART,
+	/* s-text */
+	"Listen queue depth.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	lru_interval,
+	/* tweak */	tweak_timeout,
+	/* var */	lru_interval,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	2.000,
+	/* units */	seconds,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"Grace period before object moves on LRU list.\n"
+	"Objects are only moved to the front of the LRU list if they have "
+	"not been moved there already inside this timeout period.  This "
+	"reduces the amount of lock operations necessary for LRU list "
+	"access.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	max_esi_depth,
+	/* tweak */	tweak_uint,
+	/* var */	max_esi_depth,
+	/* min */	0,
+	/* max */	none,
+	/* default */	5,
+	/* units */	levels,
+	/* flags */	00,
+	/* s-text */
+	"Maximum depth of esi:include processing.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	max_restarts,
+	/* tweak */	tweak_uint,
+	/* var */	max_restarts,
+	/* min */	0,
+	/* max */	none,
+	/* default */	4,
+	/* units */	restarts,
+	/* flags */	00,
+	/* s-text */
+	"Upper limit on how many times a request can restart.\n"
+	"Be aware that restarts are likely to cause a hit against the "
+	"backend, so don't increase thoughtlessly.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	max_retries,
+	/* tweak */	tweak_uint,
+	/* var */	max_retries,
+	/* min */	0,
+	/* max */	none,
+	/* default */	4,
+	/* units */	retries,
+	/* flags */	00,
+	/* s-text */
+	"Upper limit on how many times a backend fetch can retry.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	nuke_limit,
+	/* tweak */	tweak_uint,
+	/* var */	nuke_limit,
+	/* min */	0,
+	/* max */	none,
+	/* default */	50,
+	/* units */	allocations,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"Maximum number of objects we attempt to nuke in orderto make "
+	"space for a object body.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	pcre_match_limit,
+	/* tweak */	tweak_uint,
+	/* var */	pcre_match_limit,
+	/* min */	1,
+	/* max */	none,
+	/* default */	10000,
+	/* units */	,
+	/* flags */	00,
+	/* s-text */
+	"The limit for the  number of internal matching function calls in "
+	"a pcre_exec() execution.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	pcre_match_limit_recursion,
+	/* tweak */	tweak_uint,
+	/* var */	pcre_match_limit_recursion,
+	/* min */	1,
+	/* max */	none,
+	/* default */	10000,
+	/* units */	,
+	/* flags */	00,
+	/* s-text */
+	"The limit for the  number of internal matching function "
+	"recursions in a pcre_exec() execution.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	ping_interval,
+	/* tweak */	tweak_uint,
+	/* var */	ping_interval,
+	/* min */	0,
+	/* max */	none,
+	/* default */	3,
+	/* units */	seconds,
+	/* flags */	0| MUST_RESTART,
+	/* s-text */
+	"Interval between pings from parent to child.\n"
+	"Zero will disable pinging entirely, which makes it possible to "
+	"attach a debugger to the child.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	pipe_timeout,
+	/* tweak */	tweak_timeout,
+	/* var */	pipe_timeout,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	60.000,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"Idle timeout for PIPE sessions. If nothing have been received in "
+	"either direction for this many seconds, the session is closed.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	pool_req,
+	/* tweak */	tweak_poolparam,
+	/* var */	pool_req,
+	/* min */	none,
+	/* max */	none,
+	/* default */	10\,100\,10,
+	/* units */	none,
+	/* flags */	00,
+	/* s-text */
+	"Parameters for per worker pool request memory pool.\n"
+	"The three numbers are:\n"
+	"	min_pool	minimum size of free pool.\n"
+	"	max_pool	maximum size of free pool.\n"
+	"	max_age		max age of free element.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	pool_sess,
+	/* tweak */	tweak_poolparam,
+	/* var */	pool_sess,
+	/* min */	none,
+	/* max */	none,
+	/* default */	10,100,10,
+	/* units */	,
+	/* flags */	00,
+	/* s-text */
+	"Parameters for per worker pool session memory pool.\n"
+	"The three numbers are:\n"
+	"	min_pool	minimum size of free pool.\n"
+	"	max_pool	maximum size of free pool.\n"
+	"	max_age	max age of free element.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	pool_vbo,
+	/* tweak */	tweak_poolparam,
+	/* var */	pool_vbo,
+	/* min */	none,
+	/* max */	none,
+	/* default */	10,100,10,
+	/* units */	,
+	/* flags */	00,
+	/* s-text */
+	"Parameters for backend object fetch memory pool.\n"
+	"The three numbers are:\n"
+	"	min_pool	minimum size of free pool.\n"
+	"	max_pool	maximum size of free pool.\n"
+	"	max_age	max age of free element.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	prefer_ipv6,
+	/* tweak */	tweak_bool,
+	/* var */	prefer_ipv6,
+	/* min */	none,
+	/* max */	none,
+	/* default */	off,
+	/* units */	bool,
+	/* flags */	00,
+	/* s-text */
+	"Prefer IPv6 address when connecting to backends which have both "
+	"IPv4 and IPv6 addresses.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	rush_exponent,
+	/* tweak */	tweak_uint,
+	/* var */	rush_exponent,
+	/* min */	2,
+	/* max */	none,
+	/* default */	3,
+	/* units */	requests per request,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"How many parked request we start for each completed request on "
+	"the object.\n"
+	"NB: Even with the implict delay of delivery, this parameter "
+	"controls an exponential increase in number of worker threads.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	send_timeout,
+	/* tweak */	tweak_timeout,
+	/* var */	send_timeout,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	600.000,
+	/* units */	seconds,
+	/* flags */	0| DELAYED_EFFECT,
+	/* s-text */
+	"Send timeout for client connections. If the HTTP response hasn't "
+	"been transmitted in this many\n"
+	"seconds the session is closed.\n"
+	"See setsockopt(2) under SO_SNDTIMEO for more information.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	session_max,
+	/* tweak */	tweak_uint,
+	/* var */	session_max,
+	/* min */	1000,
+	/* max */	none,
+	/* default */	100000,
+	/* units */	sessions,
+	/* flags */	00,
+	/* s-text */
+	"Maximum number of sessions we will allocate from one pool before "
+	"just dropping connections.\n"
+	"This is mostly an anti-DoS measure, and setting it plenty high "
+	"should not hurt, as long as you have the memory for it.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+
+PARAM(
+	/* name */	shm_reclen,
+	/* tweak */	tweak_vsl_reclen,
+	/* var */	shm_reclen,
+	/* min */	16b,
+	/* max */	4084,
+	/* default */	255b,
+	/* units */	bytes,
+	/* flags */	00,
+	/* s-text */
+	"Old name for vsl_reclen, use that instead.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	shortlived,
+	/* tweak */	tweak_timeout,
+	/* var */	shortlived,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	10.000,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"Objects created with (ttl+grace+keep) shorter than this are "
+	"always put in transient storage.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	sigsegv_handler,
+	/* tweak */	tweak_bool,
+	/* var */	sigsegv_handler,
+	/* min */	none,
+	/* max */	none,
+	/* default */	off,
+	/* units */	bool,
+	/* flags */	0| MUST_RESTART,
+	/* s-text */
+	"Install a signal handler which tries to dump debug information on "
+	"segmentation and buserror faults.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	syslog_cli_traffic,
+	/* tweak */	tweak_bool,
+	/* var */	syslog_cli_traffic,
+	/* min */	none,
+	/* max */	none,
+	/* default */	on,
+	/* units */	bool,
+	/* flags */	00,
+	/* s-text */
+	"Log all CLI traffic to syslog(LOG_INFO).\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	tcp_keepalive_intvl,
+	/* tweak */	tweak_timeout,
+	/* var */	tcp_keepalive_intvl,
+	/* min */	1.000,
+	/* max */	100.000,
+	/* default */	5.000,
+	/* units */	seconds,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"The number of seconds between TCP keep-alive probes.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	tcp_keepalive_probes,
+	/* tweak */	tweak_uint,
+	/* var */	tcp_keepalive_probes,
+	/* min */	1,
+	/* max */	100,
+	/* default */	5,
+	/* units */	probes,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"The maximum number of TCP keep-alive probes to send before giving "
+	"up and killing the connection if no response is obtained from the "
+	"other end.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	tcp_keepalive_time,
+	/* tweak */	tweak_timeout,
+	/* var */	tcp_keepalive_time,
+	/* min */	1.000,
+	/* max */	7200.000,
+	/* default */	600.000,
+	/* units */	seconds,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"The number of seconds a connection needs to be idle before TCP "
+	"begins sending out keep-alive probes.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	thread_pool_add_delay,
+	/* tweak */	tweak_timeout,
+	/* var */	thread_pool_add_delay,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	0.000,
+	/* units */	seconds,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"Wait at least this long after creating a thread.\n"
+	"\n"
+	"Some (buggy) systems may need a short (sub-second) delay between "
+	"creating threads.\n"
+	"Set this to a few milliseconds if you see the 'threads_failed' "
+	"counter grow too much.\n"
+	"Setting this too high results in insuffient worker threads.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	thread_pool_destroy_delay,
+	/* tweak */	tweak_timeout,
+	/* var */	thread_pool_destroy_delay,
+	/* min */	0.010,
+	/* max */	none,
+	/* default */	1.000,
+	/* units */	seconds,
+	/* flags */	0| DELAYED_EFFECT| EXPERIMENTAL,
+	/* s-text */
+	"Wait this long after destroying a thread.\n"
+	"This controls the decay of thread pools when idle(-ish).\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	thread_pool_fail_delay,
+	/* tweak */	tweak_timeout,
+	/* var */	thread_pool_fail_delay,
+	/* min */	0.010,
+	/* max */	none,
+	/* default */	0.200,
+	/* units */	seconds,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"Wait at least this long after a failed thread creation before "
+	"trying to create another thread.\n"
+	"\n"
+	"Failure to create a worker thread is often a sign that  the end "
+	"is near, because the process is running out of some resource.  "
+	"This delay tries to not rush the end on needlessly.\n"
+	"\n"
+	"If thread creation failures are a problem, check that "
+	"thread_pool_max is not too high.\n"
+	"\n"
+	"It may also help to increase thread_pool_timeout and "
+	"thread_pool_min, to reduce the rate at which treads are destroyed "
+	"and later recreated.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	thread_pool_max,
+	/* tweak */	tweak_***,
+	/* var */	thread_pool_max,
+	/* min */	100,
+	/* max */	none,
+	/* default */	5000,
+	/* units */	threads,
+	/* flags */	0| DELAYED_EFFECT,
+	/* s-text */
+	"The maximum number of worker threads in each pool.\n"
+	"\n"
+	"Do not set this higher than you have to, since excess worker "
+	"threads soak up RAM and CPU and generally just get in the way of "
+	"getting work done.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	thread_pool_min,
+	/* tweak */	tweak_***,
+	/* var */	thread_pool_min,
+	/* min */	none,
+	/* max */	5000,
+	/* default */	100,
+	/* units */	threads,
+	/* flags */	0| DELAYED_EFFECT,
+	/* s-text */
+	"The minimum number of worker threads in each pool.\n"
+	"\n"
+	"Increasing this may help ramp up faster from low load situations "
+	"or when threads have expired.
+
+Minimum is 10 threads.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	thread_pool_stack,
+	/* tweak */	tweak_bytes,
+	/* var */	thread_pool_stack,
+	/* min */	2k,
+	/* max */	none,
+	/* default */	48k,
+	/* units */	bytes,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"Worker thread stack size.\n"
+	"This will likely be rounded up to a multiple of 4k (or whatever "
+	"the page_size might be) by the kernel.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	thread_pool_timeout,
+	/* tweak */	tweak_timeout,
+	/* var */	thread_pool_timeout,
+	/* min */	10.000,
+	/* max */	none,
+	/* default */	300.000,
+	/* units */	seconds,
+	/* flags */	0| DELAYED_EFFECT| EXPERIMENTAL,
+	/* s-text */
+	"Thread idle threshold.\n"
+	"\n"
+	"Threads in excess of thread_pool_min, which have been idle for at "
+	"least this long, will be destroyed.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	thread_pools,
+	/* tweak */	tweak_uint,
+	/* var */	thread_pools,
+	/* min */	1,
+	/* max */	none,
+	/* default */	2,
+	/* units */	pools,
+	/* flags */	0| DELAYED_EFFECT| EXPERIMENTAL,
+	/* s-text */
+	"Number of worker thread pools.\n"
+	"\n"
+	"Increasing number of worker pools decreases lock contention.\n"
+	"\n"
+	"Too many pools waste CPU and RAM resources, and more than one "
+	"pool for each CPU is probably detrimal to performance.\n"
+	"\n"
+	"Can be increased on the fly, but decreases require a restart to "
+	"take effect.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	thread_queue_limit,
+	/* tweak */	tweak_uint,
+	/* var */	thread_queue_limit,
+	/* min */	0,
+	/* max */	none,
+	/* default */	20,
+	/* units */	,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"Permitted queue length per thread-pool.\n"
+	"\n"
+	"This sets the number of requests we will queue, waiting for an "
+	"available thread.  Above this limit sessions will be dropped "
+	"instead of queued.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	thread_stats_rate,
+	/* tweak */	tweak_uint,
+	/* var */	thread_stats_rate,
+	/* min */	0,
+	/* max */	none,
+	/* default */	10,
+	/* units */	requests,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"Worker threads accumulate statistics, and dump these into the "
+	"global stats counters if the lock is free when they finish a job "
+	"(request/fetch etc.)\n"
+	"This parameters defines the maximum number of jobs a worker "
+	"thread may handle, before it is forced to dump its accumulated "
+	"stats into the global counters.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	timeout_idle,
+	/* tweak */	tweak_timeout,
+	/* var */	timeout_idle,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	5.000,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"Idle timeout for client connections.\n"
+	"A connection is considered idle, until we havereceived the full "
+	"request headers.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	timeout_linger,
+	/* tweak */	tweak_timeout,
+	/* var */	timeout_linger,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	0.050,
+	/* units */	seconds,
+	/* flags */	0| EXPERIMENTAL,
+	/* s-text */
+	"How long the worker thread lingers on an idle session before "
+	"handing it over to the waiter.\n"
+	"When sessions are reused, as much as half of all reuses happen "
+	"within the first 100 msec of the previous request completing.\n"
+	"Setting this too high results in worker threads not doing "
+	"anything for their keep, setting it too low just means that more "
+	"sessions take a detour around the waiter.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	vcc_allow_inline_c,
+	/* tweak */	tweak_bool,
+	/* var */	vcc_allow_inline_c,
+	/* min */	none,
+	/* max */	none,
+	/* default */	off,
+	/* units */	bool,
+	/* flags */	00,
+	/* s-text */
+	"Allow inline C code in VCL.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+#if 0
+PARAM(
+	/* name */	vcc_err_unref,
+	/* tweak */	tweak_bool,
+	/* var */	vcc_err_unref,
+	/* min */	none,
+	/* max */	none,
+	/* default */	on,
+	/* units */	bool,
+	/* flags */	00,
+	/* s-text */
+	"Unreferenced VCL objects result in error.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+#if 0
+PARAM(
+	/* name */	vcc_unsafe_path,
+	/* tweak */	tweak_bool,
+	/* var */	vcc_unsafe_path,
+	/* min */	none,
+	/* max */	none,
+	/* default */	on,
+	/* units */	bool,
+	/* flags */	00,
+	/* s-text */
+	"Allow '/' in vmod & include paths.\n"
+	"Allow 'import ... from ...'.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	vcl_cooldown,
+	/* tweak */	tweak_timeout,
+	/* var */	vcl_cooldown,
+	/* min */	0.000,
+	/* max */	none,
+	/* default */	600.000,
+	/* units */	seconds,
+	/* flags */	00,
+	/* s-text */
+	"How long time a VCL is kept warm after being replaced as the "
+	"active VCL.  (Granularity approximately 30 seconds.)\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	vcl_dir,
+	/* tweak */	tweak_string,
+	/* var */	vcl_dir,
+	/* min */	none,
+	/* max */	none,
+	/* default */	/opt/varnish/etc/varnish,
+	/* units */	(null),
+	/* flags */	00,
+	/* s-text */
+	"Directory from which relative VCL filenames (vcl.load and "
+	"include) are opened.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	vmod_dir,
+	/* tweak */	tweak_string,
+	/* var */	vmod_dir,
+	/* min */	none,
+	/* max */	none,
+	/* default */	/opt/varnish/lib/varnish/vmods,
+	/* units */	(null),
+	/* flags */	00,
+	/* s-text */
+	"Directory where VCL modules are to be found.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	vsl_buffer,
+	/* tweak */	tweak_vsl_buffer,
+	/* var */	vsl_buffer,
+	/* min */	267,
+	/* max */	none,
+	/* default */	4k,
+	/* units */	bytes,
+	/* flags */	00,
+	/* s-text */
+	"Bytes of (req-/backend-)workspace dedicated to buffering VSL "
+	"records.\n"
+	"Setting this too high costs memory, setting it too low will cause "
+	"more VSL flushes and likely increase lock-contention on the VSL "
+	"mutex.\n"
+	"The minimum tracks the vsl_reclen parameter + 12 bytes.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	vsl_mask,
+	/* tweak */	tweak_mask
+	/* var */	vsl_mask,
+	/* min */	none,
+	/* max */	none,
+	/* default */	-VCL_trace,-WorkThread,-Hash,-VfpAcct,
+	/* units */	,
+	/* flags */	00,
+	/* s-text */
+	"Mask individual VSL messages from being logged.\n"
+	"	default	Set default value\n"
+	"\n"
+	"Use +/- prefixe in front of VSL tag name, to mask/unmask "
+	"individual VSL messages.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	vsl_reclen,
+	/* tweak */	tweak_vsl_reclen,
+	/* var */	vsl_reclen,
+	/* min */	16b,
+	/* max */	4084b,
+	/* default */	255b,
+	/* units */	bytes,
+	/* flags */	00,
+	/* s-text */
+	"Maximum number of bytes in SHM log record.\n"
+	"The maximum tracks the vsl_buffer parameter - 12 bytes.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	vsl_space,
+	/* tweak */	tweak_bytes,
+	/* var */	vsl_space,
+	/* min */	1M,
+	/* max */	none,
+	/* default */	80M,
+	/* units */	bytes,
+	/* flags */	0| MUST_RESTART,
+	/* s-text */
+	"The amount of space to allocate for the VSL fifo buffer in the "
+	"VSM memory segment.  If you make this too small, "
+	"varnish{ncsa|log} etc will not be able to keep up.  Making it too "
+	"large just costs memory resources.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	vsm_space,
+	/* tweak */	tweak_bytes,
+	/* var */	vsm_space,
+	/* min */	1M,
+	/* max */	none,
+	/* default */	1M,
+	/* units */	bytes,
+	/* flags */	0| MUST_RESTART,
+	/* s-text */
+	"The amount of space to allocate for stats counters in the VSM "
+	"memory segment.  If you make this too small, some counters will "
+	"be invisible.  Making it too large just costs memory resources.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#if 0
+PARAM(
+	/* name */	waiter,
+	/* tweak */	tweak_waiter,
+	/* var */	waiter,
+	/* min */	none,
+	/* max */	none,
+	/* default */	kqueue (possible values: kqueue, poll),
+	/* units */	(null),
+	/* flags */	0| MUST_RESTART| WIZARD,
+	/* s-text */
+	"Select the waiter kernel interface.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+PARAM(
+	/* name */	workspace_backend,
+	/* tweak */	tweak_bytes_u,
+	/* var */	workspace_backend,
+	/* min */	1k,
+	/* max */	none,
+	/* default */	64k,
+	/* units */	bytes,
+	/* flags */	0| DELAYED_EFFECT,
+	/* s-text */
+	"Bytes of HTTP protocol workspace for backend HTTP req/resp.  If "
+	"larger than 4k, use a multiple of 4k for VM efficiency.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	workspace_client,
+	/* tweak */	tweak_bytes_u,
+	/* var */	workspace_client,
+	/* min */	9k,
+	/* max */	none,
+	/* default */	64k,
+	/* units */	bytes,
+	/* flags */	0| DELAYED_EFFECT,
+	/* s-text */
+	"Bytes of HTTP protocol workspace for clients HTTP req/resp.  If "
+	"larger than 4k, use a multiple of 4k for VM efficiency.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	workspace_session,
+	/* tweak */	tweak_bytes_u,
+	/* var */	workspace_session,
+	/* min */	0.25k,
+	/* max */	none,
+	/* default */	0.50k,
+	/* units */	bytes,
+	/* flags */	0| DELAYED_EFFECT,
+	/* s-text */
+	"Allocation size for session structure and workspace.    The "
+	"workspace is primarily used for TCP connection addresses.  If "
+	"larger than 4k, use a multiple of 4k for VM efficiency.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+PARAM(
+	/* name */	workspace_thread,
+	/* tweak */	tweak_bytes_u,
+	/* var */	workspace_thread,
+	/* min */	0.25k,
+	/* max */	8k,
+	/* default */	2k,
+	/* units */	bytes,
+	/* flags */	0| DELAYED_EFFECT,
+	/* s-text */
+	"Bytes of auxiliary workspace per thread.\n"
+	"This workspace is used for certain temporary data structures "
+	"during the operation of a worker thread.\n"
+	"One use is for the io-vectors for writing requests and responses "
+	"to sockets, having too little space will result in more writev(2) "
+	"system calls, having too much just wastes the space.\n",
+	/* l-text */	"",
+	/* func */	NULL
+)
+#endif
+/*lint -restore */



From phk at FreeBSD.org  Fri May 22 11:27:50 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 13:27:50 +0200
Subject: [master] b1ef42b Fix unterminated strings in #if 0 part of file.
Message-ID: <E1Yvl7G-0001w6-DH@project.varnish-software.com>


commit b1ef42b639d7ed4322d45fe2f07081295ea71141
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 11:26:36 2015 +0000

    Fix unterminated strings in #if 0 part of file.
    
    Gcc is really, really weird...

diff --git a/include/tbl/params.h b/include/tbl/params.h
index 78ee42b..12afdaa 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1147,9 +1147,8 @@ PARAM(
 	"The minimum number of worker threads in each pool.\n"
 	"\n"
 	"Increasing this may help ramp up faster from low load situations "
-	"or when threads have expired.
-
-Minimum is 10 threads.\n",
+	"or when threads have expired."
+	"Minimum is 10 threads.\n",
 	/* l-text */	"",
 	/* func */	NULL
 )



From phk at FreeBSD.org  Fri May 22 11:32:19 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 13:32:19 +0200
Subject: [master] cda99e3 Move variable inside #ifdef
Message-ID: <E1YvlBb-0002PX-7V@project.varnish-software.com>


commit cda99e35b5f163fe0522bb81535f056668363e8b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 11:32:10 2015 +0000

    Move variable inside #ifdef

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 210737e..9ade0ad 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -478,7 +478,6 @@ vca_acct(void *arg)
 {
 	struct listen_sock *ls;
 	double t0, now;
-	int i;
 
 	THR_SetName("cache-acceptor");
 	(void)arg;
@@ -491,6 +490,7 @@ vca_acct(void *arg)
 		vca_tcp_opt_set(ls->sock, 1);
 #ifdef HAVE_ACCEPT_FILTERS
 		if (cache_param->accept_filter) {
+			int i;
 			i = VTCP_filter_http(ls->sock);
 			if (i)
 				VSL(SLT_Error, ls->sock,



From phk at FreeBSD.org  Fri May 22 11:43:06 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 13:43:06 +0200
Subject: [master] d154379 Fix a loop doing shutdown
Message-ID: <E1YvlM2-0003P9-Bt@project.varnish-software.com>


commit d154379e13a5aacfa8aab5d6a5f4c2703eadd14f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 11:42:51 2015 +0000

    Fix a loop doing shutdown

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 9ade0ad..f4ead24 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -416,6 +416,10 @@ vca_accept_task(struct worker *wrk, void *arg)
 				VSL(SLT_Debug, ls->sock, "Too many open files");
 				vca_pace_bad();
 				break;
+			case EBADF:
+				VSL(SLT_Debug, ls->sock, "Accept failed: %s",
+				    strerror(errno));
+				return;
 			default:
 				VSL(SLT_Debug, ls->sock, "Accept failed: %s",
 				    strerror(errno));



From fgsch at lodoss.net  Fri May 22 12:40:56 2015
From: fgsch at lodoss.net (Federico Schwindt)
Date: Fri, 22 May 2015 13:40:56 +0100
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <555E5690.8090706@schokola.de>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
 <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>
 <555E5690.8090706@schokola.de>
Message-ID: <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>

Wouldn't the same apply to pcre, autoconf, automake, libtool, make, ....

On Thu, May 21, 2015 at 11:05 PM, Nils Goroll <slink at schokola.de> wrote:

> On 21/05/15 22:01, Federico Schwindt wrote:
> >
> > If you are compiling from sources (i.e. non-packaged platforms) python is
> > already required.
>
> Sorry, no. If I install a varnish-cache binary tar on a system and want to
> build
> a vmod from sources, there is no other dependency checking than that from
> the vmod.
>
> Nils
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-commit/attachments/20150522/3219a745/attachment.html>

From nils.goroll at uplex.de  Fri May 22 13:30:53 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 22 May 2015 15:30:53 +0200
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>	<CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>	<555E2417.4020104@uplex.de>	<CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>	<555E5690.8090706@schokola.de>
 <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>
Message-ID: <555F2F8D.9030402@uplex.de>

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

On 22/05/15 14:40, Federico Schwindt wrote:
> Wouldn't the same apply to pcre, autoconf, automake, libtool, make, ....

Could it be that you are bitching a bit? ;)

Trying to take your comment seriously:

- - autoconf / autogen / libtool

  autogen.sh fails when they are missing, so the error should be obvious

  you can build from a dist without them.

- - pcre: yes, vmods requiring it, should check if they have their headers
  (and at least libvmod-re does)

Nils


- -- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.ccc.de/

http://uplex.de/
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2

iQEcBAEBCAAGBQJVXy+LAAoJEB3Nj1ejhovXjVsH/1/QDU2MDzOSxWIBUtKtH99t
xcc+x1n2L4Vf7oc4VL07M//Lfv2oSoJux7OisbqnCdbWh4/e11ip0NGoKkOZxARB
zIAoq/IqCvPdDMCSkIkgSlVQLKXvz/F3j9Haa1WCIaWLVr1sekZeXltlWhH0zbNa
I3kdvcEZnZr9q9wK5SFPNJHdbu/7kv0yieLsp0ozVcjkygv0/1YC28TDRbDJY6ko
qes4TDJK+BT36M0pFu4FsdQ24T4Jr+ZIZMEdOPd7mTK13OKl32uuMArRAXkcyRfB
eUVAmV09qtahBohR5PrCDqE7q93pkrHYprQqN/pki8ORrM3yqPS9orzfOQ13jcY=
=Fjw9
-----END PGP SIGNATURE-----



From phk at FreeBSD.org  Fri May 22 14:36:06 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 16:36:06 +0200
Subject: [master] eccb26a Revert alignment to sizeof(void*). When I have a
 chance, I'll add the necessary #ifdef for the sparc64/32bit_userland thing.
Message-ID: <E1Yvo3S-0001O1-Pa@project.varnish-software.com>


commit eccb26ae2f1bfb8c842943042c880e0812b88ab5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 14:35:23 2015 +0000

    Revert alignment to sizeof(void*).  When I have a chance, I'll add
    the necessary #ifdef for the sparc64/32bit_userland thing.

diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index b7b6c25..b268082 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -130,7 +130,7 @@ void VSM_common_ageupdate(const struct vsm_sc *sc);
  * Pointer aligment magic
  */
 
-#define PALGN	    (sizeof(uintmax_t) - 1)	/* size of alignment */
+#define PALGN	    (sizeof(void *) - 1)	/* size of alignment */
 #define PAOK(p)	    (((uintptr_t)(p) & PALGN) == 0)	/* is aligned */
 #define PRNDDN(p)   ((uintptr_t)(p) & ~PALGN)		/* Round down */
 #define PRNDUP(p)   (((uintptr_t)(p) + PALGN) & ~PALGN)	/* Round up */



From phk at FreeBSD.org  Fri May 22 15:01:23 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 17:01:23 +0200
Subject: [master] 3f1704f Be more permissive about timing.
Message-ID: <E1YvoRv-0003LE-0a@project.varnish-software.com>


commit 3f1704fbca84bb5660988fe0bd8125f2d5828f29
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 15:01:07 2015 +0000

    Be more permissive about timing.

diff --git a/bin/varnishtest/tests/s00004.vtc b/bin/varnishtest/tests/s00004.vtc
index cfdc766..05d1f74 100644
--- a/bin/varnishtest/tests/s00004.vtc
+++ b/bin/varnishtest/tests/s00004.vtc
@@ -29,7 +29,7 @@ logexpect l1 -v v1 -g request {
 	expect 0 1001	Begin		req
 	expect * =	Timestamp	{Start: \S+ 0\.000000 0\.000000}
 	expect * =	Timestamp	{Req: \S+ 0\.\d+ 0\.\d+}
-	expect * =	Timestamp	{Fetch: \S+ 2\.\d+ 2\.\d+}
+	expect * =	Timestamp	{Fetch: \S+ [0-2]\.\d+ [0-2]\.\d+}
 	expect * =	Timestamp	{Process: \S+ 2\.\d+ 0\.\d+}
 	expect * =	Timestamp	{Restart: \S+ 2\.\d+ 0\.\d+}
 	expect * =	End



From phk at FreeBSD.org  Fri May 22 17:43:54 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 19:43:54 +0200
Subject: [master] 940d545 Be consistent about jit on/off flag.
Message-ID: <E1YvqzC-0000IX-Fw@project.varnish-software.com>


commit 940d545689d0d8fbd9765b10d32f8aad75111416
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 17:07:57 2015 +0000

    Be consistent about jit on/off flag.

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 0115bad..bbebef0 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -81,7 +81,7 @@ VRE_compile(const char *pattern, int options,
 		VRE_free(&v);
 		return (NULL);
 	}
-	v->re_extra = pcre_study(v->re, VRE_STUDY_JIT_COMPILE, errptr);
+	v->re_extra = pcre_study(v->re, vre__jit, errptr);
 	if (*errptr != NULL) {
 		VRE_free(&v);
 		return (NULL);



From phk at FreeBSD.org  Fri May 22 17:43:54 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 19:43:54 +0200
Subject: [master] 581f438 Reduce the default for pcre_match_limit_recursion to
 20.
Message-ID: <E1YvqzC-0000Ig-HV@project.varnish-software.com>


commit 581f438b4d6cd3e1bd04b2a3466e3f88e23027a4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 17:41:05 2015 +0000

    Reduce the default for pcre_match_limit_recursion to 20.
    
    This works (for test-case 1576 at least) on a 16k stack
    and by default we have a 48k stack.
    
    Augment the param description with some helpfull pointers.

diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index e59d38b..5ddbf37 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -520,19 +520,29 @@ struct parspec mgt_parspec[] = {
 	{ "pcre_match_limit", tweak_uint,
 		&mgt_param.vre_limits.match,
 		"1", NULL,
-		"The limit for the  number of internal matching function"
-		" calls in a pcre_exec() execution.",
+		"The limit for the number of calls to the internal match()"
+		" function in pcre_exec().\n\n"
+		"(See: PCRE_EXTRA_MATCH_LIMIT in pcre docs.)\n\n"
+		"This parameter limits how much CPU time"
+		" regular expression matching can soak up.",
 		0,
 		"10000", ""},
-
 	{ "pcre_match_limit_recursion", tweak_uint,
 		&mgt_param.vre_limits.match_recursion,
 		"1", NULL,
-		"The limit for the  number of internal matching function"
-		" recursions in a pcre_exec() execution.",
-		0,
-		"10000", ""},
-
+		"The recursion depth-limit for the internal match() function"
+		" in a pcre_exec().\n\n"
+		"(See: PCRE_EXTRA_MATCH_LIMIT_RECURSION in pcre docs.)\n\n"
+		"This puts an upper limit on the amount of stack used"
+		" by PCRE for certain classes of regular expressions.\n\n"
+		"We have set the default value low in order to"
+		" prevent crashes, at the cost of possible regexp"
+		" matching failures.\n\n"
+		"Matching failures will show up in the log as VCL_Error"
+		" messages with regexp errors -27 or -21.\n\n"
+		"Testcase r01576 can be useful when tuning this parameter.",
+		0,
+		"20", ""},
 	{ "vsl_space", tweak_bytes,
 		&mgt_param.vsl_space,
 		"1M", NULL,
diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc
index 5978504..5db70af 100644
--- a/bin/varnishtest/tests/r01576.vtc
+++ b/bin/varnishtest/tests/r01576.vtc
@@ -1,6 +1,4 @@
-varnishtest "Test certain regex fail before consuming all the stack"
-
-feature pcre_jit
+varnishtest "Test recursive regexp's fail before consuming all the stack"
 
 server s1 {
 	rxreq
@@ -11,6 +9,17 @@ server s1 {
 	txresp
 } -start
 
+# If you want to play around, uncomment the next lines and adjust
+# the length of the aaaaaaaaaaa strings below to suit your needs.
+# Better yet:  Rewrite your regexps to avoid this madness.
+
+# varnish v1 -arg "-p thread_pool_stack=48k"
+# varnish v1 -arg "-p pcre_match_limit=1000"
+# varnish v1 -arg "-p pcre_match_limit_recursion=89"
+
+# Approximate formua for FreeBSD/amd64:
+#	pcre_match_limit_recursion = thread_pool_stack * 2 - 9
+
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.url ~ "^/a((?!/.).)*$") {
@@ -19,14 +28,21 @@ varnish v1 -vcl+backend {
 	}
 } -start
 
+# This should succeed with default params and JIT/no-JIT
+client c1 {
+	txreq -url /aaaaaaaaa
+	rxresp
+} -run
+
+# PCRE_ERROR_RECURSIONLIMIT  (-21)
+# PCRE_ERROR_JIT_STACKLIMIT  (-27)
 logexpect l1 -v v1 {
-	expect * * VCL_Error "Regexp matching returned -27"
+	expect * * VCL_Error "Regexp matching returned -2[71]"
 } -start
 
+# This should fail with default params and JIT/no-JIT
 client c1 {
-	txreq -url /aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-	rxresp
-	txreq -url /aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+	txreq -url /aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 	rxresp
 } -run
 



From phk at FreeBSD.org  Fri May 22 17:43:54 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 19:43:54 +0200
Subject: [master] 341d470 Remove the "feature pcre_jit" we don't use it
 anymore.
Message-ID: <E1YvqzC-0000In-Ib@project.varnish-software.com>


commit 341d470b42eda1665359611baa2a0a3e37833439
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 17:43:26 2015 +0000

    Remove the "feature pcre_jit" we don't use it anymore.

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 9053e73..cbda7bf 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -47,7 +47,6 @@
 
 #include "vav.h"
 #include "vnum.h"
-#include "vre.h"
 #include "vtim.h"
 
 #define		MAX_TOKENS		200
@@ -599,9 +598,6 @@ cmd_feature(CMD_ARGS)
 		    getgrnam("varnish") != NULL)
 			continue;
 
-		if (!strcmp(av[i], "pcre_jit") && vre__jit)
-			continue;
-
 		vtc_log(vl, 1, "SKIPPING test, missing feature: %s", av[i]);
 		vtc_stop = 1;
 		return;



From phk at FreeBSD.org  Fri May 22 18:16:30 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 20:16:30 +0200
Subject: [master] 4871058 Be even more tolerant of timing (for the
 jenkins::arm box)
Message-ID: <E1YvrUk-0003Op-Ev@project.varnish-software.com>


commit 487105811a415c1d884dce543ac72d5c83550848
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 18:16:11 2015 +0000

    Be even more tolerant of timing (for the jenkins::arm box)

diff --git a/bin/varnishtest/tests/s00004.vtc b/bin/varnishtest/tests/s00004.vtc
index 05d1f74..2f10563 100644
--- a/bin/varnishtest/tests/s00004.vtc
+++ b/bin/varnishtest/tests/s00004.vtc
@@ -29,7 +29,7 @@ logexpect l1 -v v1 -g request {
 	expect 0 1001	Begin		req
 	expect * =	Timestamp	{Start: \S+ 0\.000000 0\.000000}
 	expect * =	Timestamp	{Req: \S+ 0\.\d+ 0\.\d+}
-	expect * =	Timestamp	{Fetch: \S+ [0-2]\.\d+ [0-2]\.\d+}
+	expect * =	Timestamp	{Fetch: \S+ [0-4]\.\d+ [0-4]\.\d+}
 	expect * =	Timestamp	{Process: \S+ 2\.\d+ 0\.\d+}
 	expect * =	Timestamp	{Restart: \S+ 2\.\d+ 0\.\d+}
 	expect * =	End



From phk at FreeBSD.org  Fri May 22 20:09:20 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 22 May 2015 22:09:20 +0200
Subject: [master] fd2cdbb Remove the magic variable used to inform
 varnishtests now defunct "feature pcre_jit".
Message-ID: <E1YvtFw-0007DL-Ld@project.varnish-software.com>


commit fd2cdbb2846b185ac27a9712d30c3d68666f9cc8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 22 20:08:29 2015 +0000

    Remove the magic variable used to inform varnishtests now defunct
    "feature pcre_jit".

diff --git a/include/vre.h b/include/vre.h
index 301e806..a59e8d7 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -51,8 +51,6 @@ typedef struct vre vre_t;
 extern const unsigned VRE_CASELESS;
 extern const unsigned VRE_NOTEMPTY;
 
-extern const int vre__jit;
-
 vre_t *VRE_compile(const char *, int, const char **, int *);
 int VRE_exec(const vre_t *code, const char *subject, int length,
     int startoffset, int options, int *ovector, int ovecsize,
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index bbebef0..94400fa 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -46,8 +46,6 @@
 #  define pcre_free_study pcre_free
 #endif
 
-const int vre__jit = VRE_STUDY_JIT_COMPILE;
-
 struct vre {
 	unsigned		magic;
 #define VRE_MAGIC		0xe83097dc
@@ -81,7 +79,7 @@ VRE_compile(const char *pattern, int options,
 		VRE_free(&v);
 		return (NULL);
 	}
-	v->re_extra = pcre_study(v->re, vre__jit, errptr);
+	v->re_extra = pcre_study(v->re, VRE_STUDY_JIT_COMPILE, errptr);
 	if (*errptr != NULL) {
 		VRE_free(&v);
 		return (NULL);



From phk at FreeBSD.org  Sat May 23 07:26:29 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 23 May 2015 09:26:29 +0200
Subject: [master] a54f85b Try to make this test run on el5-x86_64
Message-ID: <E1Yw3pF-0005pV-0h@project.varnish-software.com>


commit a54f85b4c251249192181a0fbd22f126067dcc49
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 23 07:25:56 2015 +0000

    Try to make this test run on el5-x86_64

diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc
index 5db70af..99398e3 100644
--- a/bin/varnishtest/tests/r01576.vtc
+++ b/bin/varnishtest/tests/r01576.vtc
@@ -30,7 +30,7 @@ varnish v1 -vcl+backend {
 
 # This should succeed with default params and JIT/no-JIT
 client c1 {
-	txreq -url /aaaaaaaaa
+	txreq -url /aaaaaaaa
 	rxresp
 } -run
 



From fgsch at lodoss.net  Sat May 23 09:54:08 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sat, 23 May 2015 11:54:08 +0200
Subject: [master] a2be22d Cosmetic
Message-ID: <E1Yw688-0001rg-72@project.varnish-software.com>


commit a2be22d7f0a11155d3916c5bc868e5a94f1bb3ac
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu May 21 21:27:37 2015 +0100

    Cosmetic

diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 288a1ed..82bf0aa 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -30,7 +30,7 @@ $Module std 3 Varnish Standard Module
 DESCRIPTION
 ===========
 
-Vmod_std contains basic functions which are part and parcel of Varnish,
+`vmod_std` contains basic functions which are part and parcel of Varnish,
 but which for reasons of architecture fit better in a VMOD.
 
 One particular class of functions in vmod_std is the conversions functions



From fgsch at lodoss.net  Sat May 23 09:54:08 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sat, 23 May 2015 11:54:08 +0200
Subject: [master] 2c41db8 Make this show under apropos varnish
Message-ID: <E1Yw688-0001rn-83@project.varnish-software.com>


commit 2c41db83f5c1e8387ea077a140529087f7086a64
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu May 21 21:30:21 2015 +0100

    Make this show under apropos varnish
    
    Reported by wouter-dot-hummelink-at-kpn-dot-com.
    Reword slightly while I'm here. This could use some more love.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 78c3ff2..ccfdce1 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -25,7 +25,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-$Module directors 3 Backend traffic directors
+$Module directors 3 Varnish Directors Module
 
 DESCRIPTION
 ===========
@@ -36,10 +36,7 @@ The module implements a set of basic load balancing techniques, and
 also serves as an example on how one could extend the load balancing
 capabilities of Varnish.
 
-To enable load balancing you must import this vmod (directors) in your
-VCL::
-
-  import directors;
+To enable load balancing you must import this vmod (directors).
 
 Then you define your backends. Once you have the backends declared you
 can add them to a director. This happens in executed VCL code. If you



From fgsch at lodoss.net  Sat May 23 12:30:15 2015
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sat, 23 May 2015 14:30:15 +0200
Subject: [master] 5941d35 Spelling and minor changes
Message-ID: <E1Yw8ZD-0007v8-R4@project.varnish-software.com>


commit 5941d35ce6e60bb15bea7bf5eb0d2b66717712d4
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sat May 23 11:29:04 2015 +0100

    Spelling and minor changes

diff --git a/doc/sphinx/whats-new/changes.rst b/doc/sphinx/whats-new/changes.rst
index e461f4c..2dd4a16 100644
--- a/doc/sphinx/whats-new/changes.rst
+++ b/doc/sphinx/whats-new/changes.rst
@@ -9,7 +9,7 @@ Varnish 4.1 is the continuation of the new streaming architecture seen in Varnis
 Proactive security features
 ===========================
 
-New in 4.1 is support for different kinds of privilege seperation methods,
+New in 4.1 is support for different kinds of privilege separation methods,
 collectively described as jails.
 
 On most systems, the Varnish parent process will now drop effective privileges
@@ -42,7 +42,8 @@ Output from `vcl.list`::
     available  auto/warm       0 62f5275f-a937-4df9-9fbb-c12336bdfdb8
 
 
-A single VCL's state can be chanced with the `vcl.state` call in ``varnishadm``::
+A single VCL's state can be chanced with the `vcl.state` call in
+``varnishadm``::
 
     vcl.state <configname> [auto|cold|warm]
         Force the state of the named configuration.
@@ -74,22 +75,20 @@ The ``-a`` startup argument syntax has been expanded to allow for this::
 
     $ varnishd -f /etc/varnish/default.vcl -a :6081 -a 127.0.0.1:6086,PROXY
 
-
 Both PROXY1 and PROXY2 protocols are supported on the resulting listening
 socket.
 
 For connections coming in over a PROXY socket, ``client.ip`` and
-``server.ip`` will contain the adresses given to Varnish in the PROXY
-header/preamble. (the "real" client IP.)
-
-The new VCL variables ``remote.ip`` and ``local.ip`` contains the local TCP
-connection endpoints. On non-PROXY connections these will be identical to
-``client.ip`` and ``server.ip``.
+``server.ip`` will contain the addresses given to Varnish in the PROXY
+header/preamble (the "real" IPs).
 
-An expected pattern following this is `if (std.port(local.ip) == 80) { }` in
-``vcl_recv`` to see if traffic came in over the HTTP listening socket. (so a client
-redirect to HTTPS can be served.)
+The new VCL variables ``remote.ip`` and ``local.ip`` contains the local
+TCP connection endpoints. On non-PROXY connections these will be identical
+to ``client.ip`` and ``server.ip``.
 
+An expected pattern following this is `if (std.port(local.ip) == 80) { }`
+in ``vcl_recv`` to see if traffic came in over the HTTP listening socket
+(so a client redirect to HTTPS can be served).
 
 
 VMOD backends



From phk at FreeBSD.org  Sat May 23 13:35:51 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 23 May 2015 15:35:51 +0200
Subject: [master] e05ac94 Braino.
Message-ID: <E1Yw9ah-0007bQ-Je@project.varnish-software.com>


commit e05ac94a893090707ea95bf16578b13388917165
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 23 13:35:31 2015 +0000

    Braino.
    
    Spotted by:	fgs

diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 9700546..21fe298 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -420,7 +420,7 @@ cmd_server(CMD_ARGS)
 		if (!strcmp(*av, "-dispatch")) {
 			if (strcmp(s->name, "s0")) {
 				fprintf(stderr,
-				    "server -parallel only works on s0\n");
+				    "server -dispatch only works on s0\n");
 				exit(1);
 			}
 			server_dispatch(s);



From fgsch at lodoss.net  Sun May 24 09:14:46 2015
From: fgsch at lodoss.net (Federico Schwindt)
Date: Sun, 24 May 2015 10:14:46 +0100
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <555F2F8D.9030402@uplex.de>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
 <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>
 <555E5690.8090706@schokola.de>
 <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>
 <555F2F8D.9030402@uplex.de>
Message-ID: <CAJV_h0bvYpJ3DuQ+-z-jP_f+wkkOQyLY9pjdmP4bs+WyjSA9rw@mail.gmail.com>

I'm talking about installing a varnish-cache binary tar on a system without
pcre, gcc, etc.
Heck, if you are copying the binaries to another system it might not even
start.

My concern here is that you are adding changes to Varnish for issues that
are specific to your platform and/or way of managing things and not
something general.

Following the silence procedure let's drop this here. Nobody else seems to
have an opinion.


On Fri, May 22, 2015 at 2:30 PM, Nils Goroll <nils.goroll at uplex.de> wrote:

> -----BEGIN PGP SIGNED MESSAGE-----
> Hash: SHA256
>
> On 22/05/15 14:40, Federico Schwindt wrote:
> > Wouldn't the same apply to pcre, autoconf, automake, libtool, make, ....
>
> Could it be that you are bitching a bit? ;)
>
> Trying to take your comment seriously:
>
> - - autoconf / autogen / libtool
>
>   autogen.sh fails when they are missing, so the error should be obvious
>
>   you can build from a dist without them.
>
> - - pcre: yes, vmods requiring it, should check if they have their headers
>   (and at least libvmod-re does)
>
> Nils
>
>
> - --
>
> ** * * UPLEX - Nils Goroll Systemoptimierung
>
> Scheffelstra?e 32
> 22301 Hamburg
>
> tel +49 40 28805731
> mob +49 170 2723133
> fax +49 40 42949753
>
> xmpp://slink at jabber.ccc.de/
>
> http://uplex.de/
> -----BEGIN PGP SIGNATURE-----
> Version: GnuPG v2
>
> iQEcBAEBCAAGBQJVXy+LAAoJEB3Nj1ejhovXjVsH/1/QDU2MDzOSxWIBUtKtH99t
> xcc+x1n2L4Vf7oc4VL07M//Lfv2oSoJux7OisbqnCdbWh4/e11ip0NGoKkOZxARB
> zIAoq/IqCvPdDMCSkIkgSlVQLKXvz/F3j9Haa1WCIaWLVr1sekZeXltlWhH0zbNa
> I3kdvcEZnZr9q9wK5SFPNJHdbu/7kv0yieLsp0ozVcjkygv0/1YC28TDRbDJY6ko
> qes4TDJK+BT36M0pFu4FsdQ24T4Jr+ZIZMEdOPd7mTK13OKl32uuMArRAXkcyRfB
> eUVAmV09qtahBohR5PrCDqE7q93pkrHYprQqN/pki8ORrM3yqPS9orzfOQ13jcY=
> =Fjw9
> -----END PGP SIGNATURE-----
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-commit/attachments/20150524/7c68614d/attachment.html>

From nils.goroll at uplex.de  Sun May 24 10:18:03 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Sun, 24 May 2015 12:18:03 +0200
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <CAJV_h0bvYpJ3DuQ+-z-jP_f+wkkOQyLY9pjdmP4bs+WyjSA9rw@mail.gmail.com>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>	<CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>	<555E2417.4020104@uplex.de>	<CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>	<555E5690.8090706@schokola.de>	<CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>	<555F2F8D.9030402@uplex.de>
 <CAJV_h0bvYpJ3DuQ+-z-jP_f+wkkOQyLY9pjdmP4bs+WyjSA9rw@mail.gmail.com>
Message-ID: <5561A55B.4040608@uplex.de>

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

Hi,

On 24/05/15 11:14, Federico Schwindt wrote:
> I'm talking about installing a varnish-cache binary tar on a system
> without pcre, gcc, etc. Heck, if you are copying the binaries to another
> system it might not even start.

The scenario is this:

* install a working varnish-cache distribution with all this dependencies.
  python is not a requirement for _running_ varnish-cache (only for building)
* now build a vmod. You need python to build it

> My concern here is that you are adding changes to Varnish for issues that
> are specific to your platform and/or way of managing things and not
> something general.

I appreciate your concerns. This is a simple generic issue: To build a vmod,
you need python.

> Following the silence procedure let's drop this here. Nobody else seems to
> have an opinion.

I'm fine with that, but I wanted to wrap the thing up.

Nils

- -- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.ccc.de/

http://uplex.de/
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2

iQEcBAEBCAAGBQJVYaVaAAoJEB3Nj1ejhovXee4H/RSC1814w1Nu9S/Oz2mnomFq
ippdS6+K79p96xZPkVc60r2r7R5uEq9RpJSytVxo8ySIcfNMad7Ug4O3BaAcsLM6
+jeK3HKGpi0HqtV+NHsn1saQNrffxmXWf25nFHEiGIUiUOoi8uIFS33M0RIQNnzs
59itt+znSM7h6aanOrFUb/qoWWf1zUD7TacdazJYRioilLtE6n+UhqHwtrjuTHrS
6Rdd+Z5myA8dOllK+CTPf8UIyExXRTgIWy1lQGPjBVRPYDDJqOtZLbMC8G/aR2op
jw5XEI9lfHpriZWYu3+JDD7diIows4gC1Ejqz7WLgR3LbeGNXs2x4NAJwLVeL/M=
=R2Ls
-----END PGP SIGNATURE-----



From phk at phk.freebsd.dk  Sun May 24 11:04:58 2015
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Sun, 24 May 2015 11:04:58 +0000
Subject: [master] 49712a5 vmods running vmodtool require python,
 so check for it
In-Reply-To: <5561A55B.4040608@uplex.de>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
 <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>
 <555E5690.8090706@schokola.de>
 <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>
 <555F2F8D.9030402@uplex.de>
 <CAJV_h0bvYpJ3DuQ+-z-jP_f+wkkOQyLY9pjdmP4bs+WyjSA9rw@mail.gmail.com>
 <5561A55B.4040608@uplex.de>
Message-ID: <35990.1432465498@critter.freebsd.dk>

--------

The fact that *building* a VMOD requires python is not a good reason
to make the Varnish *run-time* package depend on python.

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.



From slink at schokola.de  Sun May 24 11:45:59 2015
From: slink at schokola.de (Nils Goroll)
Date: Sun, 24 May 2015 13:45:59 +0200
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <35990.1432465498@critter.freebsd.dk>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
 <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>
 <555E5690.8090706@schokola.de>
 <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>
 <555F2F8D.9030402@uplex.de>
 <CAJV_h0bvYpJ3DuQ+-z-jP_f+wkkOQyLY9pjdmP4bs+WyjSA9rw@mail.gmail.com>
 <5561A55B.4040608@uplex.de> <35990.1432465498@critter.freebsd.dk>
Message-ID: <5561B9F7.4080004@schokola.de>

On 24/05/15 13:04, Poul-Henning Kamp wrote:
> The fact that *building* a VMOD requires python is not a good reason
> to make the Varnish *run-time* package depend on python.

Exactly. And I haven't introduced any such dependency. commit
49712a5fae5919e6e333d7164b98ef986b149c62 adds a check for python to varnish.m4
VARNISH_VMODTOOL, which is used by vmods' autoconf.




From phk at FreeBSD.org  Sun May 24 21:24:46 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 24 May 2015 23:24:46 +0200
Subject: [master] 2c022ed The changes to the backend/waiter has changed the
 dyamics of closing backend connections enough to make previously almost
 stable test-cases more unstable.
Message-ID: <E1YwdO2-0000A6-3N@project.varnish-software.com>


commit 2c022ed3ad3bfb14479e4968205827bbf8200f8e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 24 21:23:56 2015 +0000

    The changes to the backend/waiter has changed the dyamics of
    closing backend connections enough to make previously almost
    stable test-cases more unstable.
    
    Fix this one by having the server send Connection: close as appropriate.

diff --git a/bin/varnishtest/tests/c00046.vtc b/bin/varnishtest/tests/c00046.vtc
index b5baf91..52872d7 100644
--- a/bin/varnishtest/tests/c00046.vtc
+++ b/bin/varnishtest/tests/c00046.vtc
@@ -2,7 +2,7 @@ varnishtest	"Object/LRU/Stevedores with hinting and body alloc failures"
 
 server s1 {
 	rxreq
-	txresp -bodylen 1000000
+	txresp -hdr "Connection: close" -bodylen 1000000
 } -start
 
 varnish v1 \
@@ -34,7 +34,7 @@ varnish v1 -expect SMA.s2.g_space > 1000000
 server s1 -wait {
 	rxreq
 	non-fatal
-	txresp -bodylen 1000001
+	txresp -hdr "Connection: close" -bodylen 1000001
 } -start
 
 client c1 {
@@ -56,7 +56,7 @@ varnish v1 -expect SMA.s2.g_space > 1000000
 server s1 -wait {
 	rxreq
 	# non-fatal
-	txresp -bodylen 1000002
+	txresp -hdr "Connection: close" -bodylen 1000002
 } -start
 
 client c1 {



From phk at FreeBSD.org  Tue May 26 08:28:52 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 26 May 2015 10:28:52 +0200
Subject: [master] 720d0d5 Move waiter selection from param to command-line and
 pave the road for them taking parameters.
Message-ID: <E1YxAEG-0005i7-8y@project.varnish-software.com>


commit 720d0d5ae205465cd6ca6e03522579945c5fff64
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 26 08:28:10 2015 +0000

    Move waiter selection from param to command-line and pave the road
    for them taking parameters.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 1bf3355..1a5c13f 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -109,6 +109,7 @@ noinst_HEADERS = \
 	storage/storage_persistent.h \
 	waiter/waiter.h \
 	waiter/waiter_priv.h
+	waiter/mgt_waiter.h
 
 # Headers for use with vmods
 nobase_pkginclude_HEADERS = \
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 533ec80..9aa5ef3 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -57,6 +57,7 @@
 #include "vrnd.h"
 #include "vsha256.h"
 #include "vtim.h"
+#include "waiter/mgt_waiter.h"
 
 #include "compat/daemon.h"
 
@@ -186,6 +187,18 @@ usage(void)
 	    "Telnet listen address and port");
 	fprintf(stderr, FMT, "-t", "Default TTL");
 	fprintf(stderr, FMT, "-V", "version");
+	fprintf(stderr, FMT, "-W waiter", "Waiter implementation");
+#if defined(HAVE_KQUEUE)
+	fprintf(stderr, FMT, "", "  -W kqueue");
+#endif
+#if defined(HAVE_EPOLL_CTL)
+	fprintf(stderr, FMT, "", "  -W epoll");
+#endif
+#if defined(HAVE_PORT_CREATE)
+	fprintf(stderr, FMT, "", "  -W ports");
+#endif
+	fprintf(stderr, FMT, "", "  -W poll");
+
 #undef FMT
 	exit(1);
 }
@@ -438,6 +451,7 @@ main(int argc, char * const *argv)
 	const char *P_arg = NULL;
 	const char *S_arg = NULL;
 	const char *s_arg = "malloc,100m";
+	const char *W_arg = NULL;
 	int s_arg_given = 0;
 	const char *T_arg = "localhost:0";
 	char *p, *vcl = NULL;
@@ -496,7 +510,7 @@ main(int argc, char * const *argv)
 	cli_check(cli);
 
 	while ((o = getopt(argc, argv,
-	    "a:b:Cdf:Fh:i:j:l:M:n:P:p:r:S:s:T:t:Vx:")) != -1) {
+	    "a:b:Cdf:Fh:i:j:l:M:n:P:p:r:S:s:T:t:VW:x:")) != -1) {
 		/*
 		 * -j must be the first argument if specified, because
 		 * it (may) affect subsequent argument processing.
@@ -593,6 +607,9 @@ main(int argc, char * const *argv)
 			/* XXX: we should print the ident here */
 			VCS_Message("varnishd");
 			exit(0);
+		case 'W':
+			W_arg = optarg;
+			break;
 		case 'x':
 			if (!strcmp(optarg, "dumprstparam")) {
 				MCF_DumpRstParam();
@@ -708,6 +725,8 @@ main(int argc, char * const *argv)
 
 	HSH_config(h_arg);
 
+	Wait_config(W_arg);
+
 	mgt_SHM_Init();
 
 	AZ(VSB_finish(vident));
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index 5ddbf37..c97d3ff 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -35,7 +35,6 @@
 #include "common/params.h"
 
 #include "mgt/mgt_param.h"
-#include "waiter/waiter.h"
 
 
 #define MEMPOOL_TEXT							\
@@ -370,11 +369,6 @@ struct parspec mgt_parspec[] = {
 		"more sessions take a detour around the waiter.",
 		EXPERIMENTAL,
 		"0.050", "seconds" },
-	{ "waiter", tweak_waiter, NULL,
-		NULL, NULL,
-		"Select the waiter kernel interface.",
-		WIZARD | MUST_RESTART,
-		WAITER_DEFAULT, NULL },
 	{ "ban_dups", tweak_bool, &mgt_param.ban_dups,
 		NULL, NULL,
 		"Eliminate older identical bans when new bans are created."
diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c
index f75a5b5..b5f653c 100644
--- a/bin/varnishd/mgt/mgt_param_tweak.c
+++ b/bin/varnishd/mgt/mgt_param_tweak.c
@@ -43,7 +43,6 @@
 #include "common/params.h"
 
 #include "mgt/mgt_param.h"
-#include "waiter/waiter.h"
 #include "vav.h"
 #include "vnum.h"
 
@@ -381,17 +380,6 @@ tweak_string(struct vsb *vsb, const struct parspec *par, const char *arg)
 /*--------------------------------------------------------------------*/
 
 int
-tweak_waiter(struct vsb *vsb, const struct parspec *par, const char *arg)
-{
-
-	/* XXX should have tweak_generic_string */
-	(void)par;
-	return (Wait_Argument(vsb, arg));
-}
-
-/*--------------------------------------------------------------------*/
-
-int
 tweak_poolparam(struct vsb *vsb, const struct parspec *par, const char *arg)
 {
 	volatile struct poolparam *pp, px;
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index 3373d0c..5965c8a 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -38,6 +38,7 @@
 
 #include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
+#include "waiter/mgt_waiter.h"
 
 int
 Wait_Enter(const struct waiter *w, struct waited *wp)
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index 0452fe9..01f568c 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -43,6 +43,7 @@
 
 #include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
+#include "waiter/mgt_waiter.h"
 #include "vtim.h"
 #include "vfil.h"
 
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index 34aae98..b52ded0 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -42,6 +42,7 @@
 
 #include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
+#include "waiter/mgt_waiter.h"
 #include "vtim.h"
 
 #define NKEV	256
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index b54d695..210cd78 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -38,6 +38,7 @@
 
 #include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
+#include "waiter/mgt_waiter.h"
 #include "vtim.h"
 
 struct vwp {
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index a41c8fa..eb3ad4a 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -45,6 +45,7 @@
 
 #include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
+#include "waiter/mgt_waiter.h"
 #include "vtim.h"
 
 #define MAX_EVENTS 256
diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c
index 3f344fd..18b3269 100644
--- a/bin/varnishd/waiter/mgt_waiter.c
+++ b/bin/varnishd/waiter/mgt_waiter.c
@@ -33,59 +33,35 @@
 #include <unistd.h>
 #include <string.h>
 
-#include "common/common.h"
+#include "mgt/mgt.h"
+#include "waiter/mgt_waiter.h"
 
-#include "waiter/waiter.h"
-#include "waiter/waiter_priv.h"
-
-static const struct waiter_impl *const waiter_impls[] = {
+static const struct choice waiter_choice[] = {
     #if defined(HAVE_KQUEUE)
-	&waiter_kqueue,
+	{ "kqueue",	&waiter_kqueue },
     #endif
     #if defined(HAVE_EPOLL_CTL)
-	&waiter_epoll,
+	{ "epoll",	&waiter_epoll },
     #endif
 #if 0
     #if defined(HAVE_PORT_CREATE)
-	&waiter_ports,
+	{ "ports",	&waiter_ports },
     #endif
 #endif
-	&waiter_poll,
-	NULL,
+	{ "poll",	&waiter_poll },
+	{ NULL,		NULL}
 };
 
 struct waiter_impl const *waiter;
 
-int
-Wait_Argument(struct vsb *vsb, const char *arg)
+void
+Wait_config(const char *arg)
 {
-	int i;
 
 	ASSERT_MGT();
 
-	if (arg == NULL) {
-		if (waiter == NULL)
-			VSB_printf(vsb, "default");
-		else
-			VSB_printf(vsb, "%s", waiter->name);
-
-		VSB_printf(vsb, " (possible values: ");
-		for (i = 0; waiter_impls[i] != NULL; i++)
-			VSB_printf(vsb, "%s%s", i == 0 ? "" : ", ",
-			    waiter_impls[i]->name);
-		VSB_printf(vsb, ")");
-		return(0);
-	}
-	if (!strcmp(arg, WAITER_DEFAULT)) {
-		waiter = waiter_impls[0];
-		return(0);
-	}
-	for (i = 0; waiter_impls[i]; i++) {
-		if (!strcmp(arg, waiter_impls[i]->name)) {
-			waiter = waiter_impls[i];
-			return(0);
-		}
-	}
-	VSB_printf(vsb, "Unknown waiter");
-	return (-1);
+	if (arg != NULL)
+		waiter = pick(waiter_choice, arg, "waiter");
+	else
+		waiter = waiter_choice[0].ptr;
 }
diff --git a/bin/varnishd/waiter/mgt_waiter.h b/bin/varnishd/waiter/mgt_waiter.h
new file mode 100644
index 0000000..0106427
--- /dev/null
+++ b/bin/varnishd/waiter/mgt_waiter.h
@@ -0,0 +1,51 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Private interfaces
+ */
+
+struct waiter_impl;
+
+/* mgt_waiter.c */
+extern struct waiter_impl const * waiter;
+
+#if defined(HAVE_EPOLL_CTL)
+extern const struct waiter_impl waiter_epoll;
+#endif
+
+#if defined(HAVE_KQUEUE)
+extern const struct waiter_impl waiter_kqueue;
+#endif
+
+#if defined(HAVE_PORT_CREATE)
+extern const struct waiter_impl waiter_ports;
+#endif
+
+extern const struct waiter_impl waiter_poll;
+
+void Wait_config(const char *arg);
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
index 5b5620b..c211fd6 100644
--- a/bin/varnishd/waiter/waiter.h
+++ b/bin/varnishd/waiter/waiter.h
@@ -61,6 +61,3 @@ int Wait_Enter(const struct waiter *, struct waited *);
 struct waiter *Waiter_New(waiter_handle_f *, volatile double *timeout);
 void Waiter_Destroy(struct waiter **);
 const char *Waiter_GetName(void);
-
-/* mgt_waiter.c */
-int Wait_Argument(struct vsb *vsb, const char *arg);
diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h
index ed369c4..61ff970 100644
--- a/bin/varnishd/waiter/waiter_priv.h
+++ b/bin/varnishd/waiter/waiter_priv.h
@@ -63,20 +63,3 @@ struct waiter_impl {
 	waiter_inject_f		*inject;
 	size_t			size;
 };
-
-/* mgt_waiter.c */
-extern struct waiter_impl const * waiter;
-
-#if defined(HAVE_EPOLL_CTL)
-extern const struct waiter_impl waiter_epoll;
-#endif
-
-#if defined(HAVE_KQUEUE)
-extern const struct waiter_impl waiter_kqueue;
-#endif
-
-#if defined(HAVE_PORT_CREATE)
-extern const struct waiter_impl waiter_ports;
-#endif
-
-extern const struct waiter_impl waiter_poll;
diff --git a/bin/varnishtest/tests/b00008.vtc b/bin/varnishtest/tests/b00008.vtc
index dbadb1b..2448aa4 100644
--- a/bin/varnishtest/tests/b00008.vtc
+++ b/bin/varnishtest/tests/b00008.vtc
@@ -24,8 +24,6 @@ varnish v1 -start
 
 varnish v1 -cliok "help"
 
-varnish v1 -cliok "param.set waiter poll"
-
 varnish v1 -clierr 106 "param.set waiter HASH(0x8839c4c)"
 
 varnish v1 -cliok "param.set cli_limit 128"
diff --git a/bin/varnishtest/tests/b00009.vtc b/bin/varnishtest/tests/b00009.vtc
index f30fd01..cbd4b1e 100644
--- a/bin/varnishtest/tests/b00009.vtc
+++ b/bin/varnishtest/tests/b00009.vtc
@@ -5,7 +5,7 @@ server s1 {
 	txresp -hdr "Connection: close" -body "012345\n"
 } -start
 
-varnish v1 -arg "-p waiter=poll" -vcl+backend {} -start
+varnish v1 -arg "-Wpoll" -vcl+backend {} -start
 
 client c1 {
 	txreq -url "/"
diff --git a/bin/varnishtest/tests/b00048.vtc b/bin/varnishtest/tests/b00048.vtc
index ab9089c..e2a4b95 100644
--- a/bin/varnishtest/tests/b00048.vtc
+++ b/bin/varnishtest/tests/b00048.vtc
@@ -10,7 +10,7 @@ server s0 {
 	expect_close
 } -dispatch
 
-varnish v1 -arg "-p waiter=poll" -vcl+backend {
+varnish v1 -arg "-Wpoll" -vcl+backend {
 	sub vcl_recv {
 		return (pass);
 	}



From phk at FreeBSD.org  Tue May 26 08:40:37 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 26 May 2015 10:40:37 +0200
Subject: [master] 330cd26 Concentrate waiter stuff in waiter includes
Message-ID: <E1YxAPd-0007E6-Ff@project.varnish-software.com>


commit 330cd262e4d434bb87947fca44872b798bcc431a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 26 08:40:25 2015 +0000

    Concentrate waiter stuff in waiter includes

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 1a0dc03..ba45bf2 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -41,6 +41,8 @@
 #include "vapi/vsl_int.h"
 #include "vapi/vsm_int.h"
 
+#include "waiter/waiter.h"
+
 #include <sys/socket.h>
 
 #include <pthread.h>
@@ -366,19 +368,6 @@ struct lru {
 	unsigned		n_objcore;
 };
 
-/* Connection waiter -------------------------------------------------
- * Describing a file-descriptor/connection being waited on
- */
-
-struct waited {
-	unsigned		magic;
-#define WAITED_MAGIC		0x1743992d
-	int			fd;
-	void			*ptr;
-	double			idle;
-	VTAILQ_ENTRY(waited)	list;
-};
-
 /* Stored object -----------------------------------------------------
  * Pointer to a stored object, and the methods it supports
  */
diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index 44b6351..16b989e 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -41,7 +41,6 @@
 #include "cache_backend.h"
 #include "vtcp.h"
 #include "vsa.h"
-#include "waiter/waiter.h"
 #include "vtim.h"
 
 struct tcp_pool {
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index ee1091c..6f59a5e 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -47,7 +47,6 @@
 #include "storage/storage.h"
 #include "vcl.h"
 #include "vcli_priv.h"
-#include "waiter/waiter.h"
 
 /*
  * The panic string is constructed in memory, then copied to the
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 37a6109..950023c 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -47,7 +47,6 @@
 
 #include "cache.h"
 
-#include "waiter/waiter.h"
 #include "vsa.h"
 #include "vtcp.h"
 #include "vtim.h"
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index 5965c8a..5e76eb0 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -36,7 +36,6 @@
 
 #include "cache/cache.h"
 
-#include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
 #include "waiter/mgt_waiter.h"
 
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index 01f568c..9302add 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -41,7 +41,6 @@
 
 #include "cache/cache.h"
 
-#include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
 #include "waiter/mgt_waiter.h"
 #include "vtim.h"
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index b52ded0..6c55ac9 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -40,7 +40,6 @@
 
 #include "cache/cache.h"
 
-#include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
 #include "waiter/mgt_waiter.h"
 #include "vtim.h"
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index 210cd78..fba970b 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -36,7 +36,6 @@
 
 #include "cache/cache.h"
 
-#include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
 #include "waiter/mgt_waiter.h"
 #include "vtim.h"
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index eb3ad4a..d02f302 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -43,7 +43,6 @@
 
 #include "cache/cache.h"
 
-#include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
 #include "waiter/mgt_waiter.h"
 #include "vtim.h"
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
index c211fd6..500b36a 100644
--- a/bin/varnishd/waiter/waiter.h
+++ b/bin/varnishd/waiter/waiter.h
@@ -42,9 +42,21 @@
  * Public interfaces
  */
 
-struct waited;
 struct waiter;
 
+/* Connection waiter -------------------------------------------------
+ * Describing a file-descriptor/connection being waited on
+ */
+
+struct waited {
+	unsigned		magic;
+#define WAITED_MAGIC		0x1743992d
+	int			fd;
+	void			*ptr;
+	double			idle;
+	VTAILQ_ENTRY(waited)	list;
+};
+
 enum wait_event {
 	WAITER_REMCLOSE,
 	WAITER_TIMEOUT,



From phk at FreeBSD.org  Tue May 26 08:44:41 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 26 May 2015 10:44:41 +0200
Subject: [master] d0b4ba4 Add a missing line continuation
Message-ID: <E1YxATZ-0007eu-LJ@project.varnish-software.com>


commit d0b4ba4485d1e941e8053bdee5745bc478285cb0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 26 08:41:34 2015 +0000

    Add a missing line continuation

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 1a5c13f..34425f1 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -108,7 +108,7 @@ noinst_HEADERS = \
 	storage/storage.h \
 	storage/storage_persistent.h \
 	waiter/waiter.h \
-	waiter/waiter_priv.h
+	waiter/waiter_priv.h \
 	waiter/mgt_waiter.h
 
 # Headers for use with vmods



From phk at FreeBSD.org  Tue May 26 08:44:41 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 26 May 2015 10:44:41 +0200
Subject: [master] 26f6726 Be consistent about filenames:  mgt_ is a prefix
Message-ID: <E1YxATZ-0007f0-Mm@project.varnish-software.com>


commit 26f67269a23ab67a83d30bb436784424160da1af
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 26 08:44:24 2015 +0000

    Be consistent about filenames:  mgt_ is a prefix

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 34425f1..ac3b216 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -54,7 +54,7 @@ varnishd_SOURCES = \
 	common/common_vsc.c \
 	hash/hash_classic.c \
 	hash/hash_critbit.c \
-	hash/hash_mgt.c \
+	hash/mgt_hash.c \
 	hash/hash_simple_list.c \
 	http1/cache_http1_deliver.c \
 	http1/cache_http1_fetch.c \
@@ -81,12 +81,12 @@ varnishd_SOURCES = \
 	mgt/mgt_vcl.c \
 	proxy/cache_proxy_proto.c \
 	storage/stevedore.c \
-	storage/stevedore_mgt.c \
+	storage/mgt_stevedore.c \
 	storage/stevedore_utils.c \
 	storage/storage_file.c \
 	storage/storage_malloc.c \
 	storage/storage_persistent.c \
-	storage/storage_persistent_mgt.c \
+	storage/mgt_storage_persistent.c \
 	storage/storage_persistent_silo.c \
 	storage/storage_persistent_subr.c \
 	storage/storage_umem.c \
diff --git a/bin/varnishd/hash/hash_mgt.c b/bin/varnishd/hash/hash_mgt.c
deleted file mode 100644
index a8cc352..0000000
--- a/bin/varnishd/hash/hash_mgt.c
+++ /dev/null
@@ -1,84 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-
-#include "mgt/mgt.h"
-#include "common/heritage.h"
-
-#include "hash/hash_slinger.h"
-#include "vav.h"
-
-static const struct choice hsh_choice[] = {
-	{ "classic",		&hcl_slinger },
-	{ "simple",		&hsl_slinger },
-	{ "simple_list",	&hsl_slinger },	/* backwards compat */
-	{ "critbit",		&hcb_slinger },
-	{ NULL,			NULL }
-};
-
-/*--------------------------------------------------------------------*/
-
-void
-HSH_config(const char *h_arg)
-{
-	char **av;
-	int ac;
-	const struct hash_slinger *hp;
-
-	ASSERT_MGT();
-	av = VAV_Parse(h_arg, NULL, ARGV_COMMA);
-	AN(av);
-
-	if (av[0] != NULL)
-		ARGV_ERR("%s\n", av[0]);
-
-	if (av[1] == NULL)
-		ARGV_ERR("-h argument is empty\n");
-
-	for (ac = 0; av[ac + 2] != NULL; ac++)
-		continue;
-
-	hp = pick(hsh_choice, av[1], "hash");
-	CHECK_OBJ_NOTNULL(hp, SLINGER_MAGIC);
-	VSB_printf(vident, ",-h%s", av[1]);
-	heritage.hash = hp;
-	if (hp->init != NULL)
-		hp->init(ac, av + 2);
-	else if (ac > 0)
-		ARGV_ERR("Hash method \"%s\" takes no arguments\n",
-		    hp->name);
-	/* NB: Don't free av, the hasher is allowed to keep it. */
-}
diff --git a/bin/varnishd/hash/mgt_hash.c b/bin/varnishd/hash/mgt_hash.c
new file mode 100644
index 0000000..a8cc352
--- /dev/null
+++ b/bin/varnishd/hash/mgt_hash.c
@@ -0,0 +1,84 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+#include "common/heritage.h"
+
+#include "hash/hash_slinger.h"
+#include "vav.h"
+
+static const struct choice hsh_choice[] = {
+	{ "classic",		&hcl_slinger },
+	{ "simple",		&hsl_slinger },
+	{ "simple_list",	&hsl_slinger },	/* backwards compat */
+	{ "critbit",		&hcb_slinger },
+	{ NULL,			NULL }
+};
+
+/*--------------------------------------------------------------------*/
+
+void
+HSH_config(const char *h_arg)
+{
+	char **av;
+	int ac;
+	const struct hash_slinger *hp;
+
+	ASSERT_MGT();
+	av = VAV_Parse(h_arg, NULL, ARGV_COMMA);
+	AN(av);
+
+	if (av[0] != NULL)
+		ARGV_ERR("%s\n", av[0]);
+
+	if (av[1] == NULL)
+		ARGV_ERR("-h argument is empty\n");
+
+	for (ac = 0; av[ac + 2] != NULL; ac++)
+		continue;
+
+	hp = pick(hsh_choice, av[1], "hash");
+	CHECK_OBJ_NOTNULL(hp, SLINGER_MAGIC);
+	VSB_printf(vident, ",-h%s", av[1]);
+	heritage.hash = hp;
+	if (hp->init != NULL)
+		hp->init(ac, av + 2);
+	else if (ac > 0)
+		ARGV_ERR("Hash method \"%s\" takes no arguments\n",
+		    hp->name);
+	/* NB: Don't free av, the hasher is allowed to keep it. */
+}
diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c
new file mode 100644
index 0000000..109290f
--- /dev/null
+++ b/bin/varnishd/storage/mgt_stevedore.c
@@ -0,0 +1,211 @@
+/*-
+ * Copyright (c) 2007-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dag-Erling Sm?rgav <des at des.no>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * STEVEDORE: one who works at or is responsible for loading and
+ * unloading ships in port.  Example: "on the wharves, stevedores were
+ * unloading cargo from the far corners of the world." Origin: Spanish
+ * estibador, from estibar to pack.  First Known Use: 1788
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+#include "vcli_priv.h"
+#include "mgt/mgt_cli.h"
+
+#include "storage/storage.h"
+#include "vav.h"
+
+struct stevedore_head stv_stevedores =
+    VTAILQ_HEAD_INITIALIZER(stv_stevedores);
+
+struct stevedore *stv_transient;
+
+/*--------------------------------------------------------------------*/
+
+static void
+stv_cli_list(struct cli *cli, const char * const *av, void *priv)
+{
+	struct stevedore *stv;
+
+	ASSERT_MGT();
+	(void)av;
+	(void)priv;
+	VCLI_Out(cli, "Storage devices:\n");
+	stv = stv_transient;
+		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
+	VTAILQ_FOREACH(stv, &stv_stevedores, list)
+		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
+}
+
+/*--------------------------------------------------------------------*/
+
+struct cli_proto cli_stv[] = {
+	{ "storage.list", "storage.list", "\tList storage devices.",
+	    0, 0, "", stv_cli_list },
+	{ NULL}
+};
+
+/*--------------------------------------------------------------------
+ */
+
+static void
+smp_fake_init(struct stevedore *parent, int ac, char * const *av)
+{
+
+	(void)parent;
+	(void)ac;
+	(void)av;
+	ARGV_ERR(
+	    "-spersistent has been deprecated, please see:\n"
+	    "  https://www.varnish-cache.org/docs/trunk/phk/persistent.html\n"
+	    "for details.\n"
+	);
+}
+
+
+static const struct stevedore smp_fake_stevedore = {
+	.magic = STEVEDORE_MAGIC,
+	.name = "deprecated_persistent",
+	.init = smp_fake_init,
+};
+
+
+/*--------------------------------------------------------------------
+ * Parse a stevedore argument on the form:
+ *	[ name '=' ] strategy [ ',' arg ] *
+ */
+
+static const struct choice STV_choice[] = {
+	{ "file",			&smf_stevedore },
+	{ "malloc",			&sma_stevedore },
+	{ "deprecated_persistent",	&smp_stevedore },
+	{ "persistent",			&smp_fake_stevedore },
+#ifdef HAVE_LIBUMEM
+	{ "umem",			&smu_stevedore },
+#endif
+	{ NULL,		NULL }
+};
+
+void
+STV_Config(const char *spec)
+{
+	char **av;
+	const char *p, *q;
+	struct stevedore *stv;
+	const struct stevedore *stv2;
+	int ac, l;
+	static unsigned seq = 0;
+
+	ASSERT_MGT();
+	p = strchr(spec, '=');
+	q = strchr(spec, ',');
+	if (p != NULL && (q == NULL || q > p)) {
+		av = VAV_Parse(p + 1, NULL, ARGV_COMMA);
+	} else {
+		av = VAV_Parse(spec, NULL, ARGV_COMMA);
+		p = NULL;
+	}
+	AN(av);
+
+	if (av[0] != NULL)
+		ARGV_ERR("%s\n", av[0]);
+
+	if (av[1] == NULL)
+		ARGV_ERR("-s argument lacks strategy {malloc, file, ...}\n");
+
+	for (ac = 0; av[ac + 2] != NULL; ac++)
+		continue;
+
+	stv2 = pick(STV_choice, av[1], "storage");
+	AN(stv2);
+
+	/* Append strategy to ident string */
+	VSB_printf(vident, ",-s%s", av[1]);
+
+	av += 2;
+
+	CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC);
+	ALLOC_OBJ(stv, STEVEDORE_MAGIC);
+	AN(stv);
+
+	*stv = *stv2;
+	AN(stv->name);
+
+	if (p == NULL)
+		bprintf(stv->ident, "s%u", seq++);
+	else {
+		l = p - spec;
+		if (l > sizeof stv->ident - 1)
+			l = sizeof stv->ident - 1;
+		bprintf(stv->ident, "%.*s", l, spec);
+	}
+
+	VTAILQ_FOREACH(stv2, &stv_stevedores, list) {
+		if (strcmp(stv2->ident, stv->ident))
+			continue;
+		ARGV_ERR("(-s%s=%s) already defined once\n",
+		    stv->ident, stv->name);
+	}
+
+	if (stv->init != NULL)
+		stv->init(stv, ac, av);
+	else if (ac != 0)
+		ARGV_ERR("(-s%s) too many arguments\n", stv->name);
+
+	AN(stv->alloc);
+	if (stv->allocobj == NULL)
+		stv->allocobj = stv_default_allocobj;
+
+	if (!strcmp(stv->ident, TRANSIENT_STORAGE)) {
+		stv->transient = 1;
+		AZ(stv_transient);
+		stv_transient = stv;
+	} else {
+		VTAILQ_INSERT_TAIL(&stv_stevedores, stv, list);
+	}
+	/* NB: Do not free av, stevedore gets to keep it */
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+STV_Config_Transient(void)
+{
+
+	ASSERT_MGT();
+
+	if (stv_transient == NULL)
+		STV_Config(TRANSIENT_STORAGE "=malloc");
+}
+
+/*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/storage/mgt_storage_persistent.c b/bin/varnishd/storage/mgt_storage_persistent.c
new file mode 100644
index 0000000..76d6436
--- /dev/null
+++ b/bin/varnishd/storage/mgt_storage_persistent.c
@@ -0,0 +1,209 @@
+/*-
+ * Copyright (c) 2008-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Persistent storage method
+ *
+ * XXX: Before we start the client or maybe after it stops, we should give the
+ * XXX: stevedores a chance to examine their storage for consistency.
+ *
+ * XXX: Do we ever free the LRU-lists ?
+ */
+
+#include "config.h"
+
+#include <sys/mman.h>
+
+#include <errno.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache/cache.h"
+#include "storage/storage.h"
+
+#include "vsha256.h"
+
+#include "storage/storage_persistent.h"
+
+#ifndef MAP_NOCORE
+#define MAP_NOCORE 0 /* XXX Linux */
+#endif
+
+#ifndef MAP_NOSYNC
+#define MAP_NOSYNC 0 /* XXX Linux */
+#endif
+
+/*--------------------------------------------------------------------
+ * Calculate cleaner metrics from silo dimensions
+ */
+
+static void
+smp_metrics(struct smp_sc *sc)
+{
+
+	/*
+	 * We do not want to loose too big chunks of the silos
+	 * content when we are forced to clean a segment.
+	 *
+	 * For now insist that a segment covers no more than 1% of the silo.
+	 *
+	 * XXX: This should possibly depend on the size of the silo so
+	 * XXX: trivially small silos do not run into trouble along
+	 * XXX: the lines of "one object per segment".
+	 */
+
+	sc->min_nseg = 10;
+	sc->max_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->min_nseg;
+
+	fprintf(stderr, "min_nseg = %u, max_segl = %ju\n",
+	    sc->min_nseg, (uintmax_t)sc->max_segl);
+
+	/*
+	 * The number of segments are limited by the size of the segment
+	 * table(s) and from that follows the minimum size of a segmement.
+	 */
+
+	sc->max_nseg = smp_stuff_len(sc, SMP_SEG1_STUFF) / sc->min_nseg;
+	sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
+
+	while (sc->min_segl < sizeof(struct object)) {
+		sc->max_nseg /= 2;
+		sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
+	}
+
+	fprintf(stderr, "max_nseg = %u, min_segl = %ju\n",
+	    sc->max_nseg, (uintmax_t)sc->min_segl);
+
+	/*
+	 * Set our initial aim point at the exponential average of the
+	 * two extremes.
+	 *
+	 * XXX: This is a pretty arbitrary choice, but having no idea
+	 * XXX: object count, size distribution or ttl pattern at this
+	 * XXX: point, we have to do something.
+	 */
+
+	sc->aim_nseg =
+	   (unsigned) exp((log(sc->min_nseg) + log(sc->max_nseg))*.5);
+	sc->aim_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->aim_nseg;
+
+	fprintf(stderr, "aim_nseg = %u, aim_segl = %ju\n",
+	    sc->aim_nseg, (uintmax_t)sc->aim_segl);
+
+	/*
+	 * How much space in the free reserve pool ?
+	 */
+	sc->free_reserve = sc->aim_segl * 10;
+
+	fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
+}
+
+/*--------------------------------------------------------------------
+ * Set up persistent storage silo in the master process.
+ */
+
+void
+smp_mgt_init(struct stevedore *parent, int ac, char * const *av)
+{
+	struct smp_sc		*sc;
+	struct smp_sign		sgn;
+	void *target;
+	int i;
+
+	ASSERT_MGT();
+
+	AZ(av[ac]);
+
+	/* Necessary alignment. See also smp_object::__filler__ */
+	assert(sizeof(struct smp_object) % 8 == 0);
+
+#define SIZOF(foo)       fprintf(stderr, \
+    "sizeof(%s) = %zu = 0x%zx\n", #foo, sizeof(foo), sizeof(foo));
+	SIZOF(struct smp_ident);
+	SIZOF(struct smp_sign);
+	SIZOF(struct smp_segptr);
+	SIZOF(struct smp_object);
+#undef SIZOF
+
+	/* See comments in storage_persistent.h */
+	assert(sizeof(struct smp_ident) == SMP_IDENT_SIZE);
+
+	/* Allocate softc */
+	ALLOC_OBJ(sc, SMP_SC_MAGIC);
+	XXXAN(sc);
+	sc->parent = parent;
+	sc->fd = -1;
+	VTAILQ_INIT(&sc->segments);
+
+	/* Argument processing */
+	if (ac != 2)
+		ARGV_ERR("(-spersistent) wrong number of arguments\n");
+
+	i = STV_GetFile(av[0], &sc->fd, &sc->filename, "-spersistent");
+	if (i == 2)
+		ARGV_ERR("(-spersistent) need filename (not directory)\n");
+
+	sc->align = sizeof(void*) * 2;
+	sc->granularity = getpagesize();
+	sc->mediasize = STV_FileSize(sc->fd, av[1], &sc->granularity,
+	    "-spersistent");
+
+	AZ(ftruncate(sc->fd, sc->mediasize));
+
+	/* Try to determine correct mmap address */
+	i = read(sc->fd, &sgn, sizeof sgn);
+	assert(i == sizeof sgn);
+	if (!strcmp(sgn.ident, "SILO"))
+		target = (void*)(uintptr_t)sgn.mapped;
+	else
+		target = NULL;
+
+	sc->base = (void*)mmap(target, sc->mediasize, PROT_READ|PROT_WRITE,
+	    MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, 0);
+
+	if (sc->base == MAP_FAILED)
+		ARGV_ERR("(-spersistent) failed to mmap (%s)\n",
+		    strerror(errno));
+
+	smp_def_sign(sc, &sc->idn, 0, "SILO");
+	sc->ident = SIGN_DATA(&sc->idn);
+
+	i = smp_valid_silo(sc);
+	if (i) {
+		printf("Warning SILO (%s) not reloaded (reason=%d)\n",
+		    sc->filename, i);
+		smp_newsilo(sc);
+	}
+	AZ(smp_valid_silo(sc));
+
+	smp_metrics(sc);
+
+	parent->priv = sc;
+
+	/* XXX: only for sendfile I guess... */
+	mgt_child_inherit(sc->fd, "storage_persistent");
+}
diff --git a/bin/varnishd/storage/stevedore_mgt.c b/bin/varnishd/storage/stevedore_mgt.c
deleted file mode 100644
index 109290f..0000000
--- a/bin/varnishd/storage/stevedore_mgt.c
+++ /dev/null
@@ -1,211 +0,0 @@
-/*-
- * Copyright (c) 2007-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Dag-Erling Sm?rgav <des at des.no>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * STEVEDORE: one who works at or is responsible for loading and
- * unloading ships in port.  Example: "on the wharves, stevedores were
- * unloading cargo from the far corners of the world." Origin: Spanish
- * estibador, from estibar to pack.  First Known Use: 1788
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "mgt/mgt.h"
-#include "vcli_priv.h"
-#include "mgt/mgt_cli.h"
-
-#include "storage/storage.h"
-#include "vav.h"
-
-struct stevedore_head stv_stevedores =
-    VTAILQ_HEAD_INITIALIZER(stv_stevedores);
-
-struct stevedore *stv_transient;
-
-/*--------------------------------------------------------------------*/
-
-static void
-stv_cli_list(struct cli *cli, const char * const *av, void *priv)
-{
-	struct stevedore *stv;
-
-	ASSERT_MGT();
-	(void)av;
-	(void)priv;
-	VCLI_Out(cli, "Storage devices:\n");
-	stv = stv_transient;
-		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
-	VTAILQ_FOREACH(stv, &stv_stevedores, list)
-		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
-}
-
-/*--------------------------------------------------------------------*/
-
-struct cli_proto cli_stv[] = {
-	{ "storage.list", "storage.list", "\tList storage devices.",
-	    0, 0, "", stv_cli_list },
-	{ NULL}
-};
-
-/*--------------------------------------------------------------------
- */
-
-static void
-smp_fake_init(struct stevedore *parent, int ac, char * const *av)
-{
-
-	(void)parent;
-	(void)ac;
-	(void)av;
-	ARGV_ERR(
-	    "-spersistent has been deprecated, please see:\n"
-	    "  https://www.varnish-cache.org/docs/trunk/phk/persistent.html\n"
-	    "for details.\n"
-	);
-}
-
-
-static const struct stevedore smp_fake_stevedore = {
-	.magic = STEVEDORE_MAGIC,
-	.name = "deprecated_persistent",
-	.init = smp_fake_init,
-};
-
-
-/*--------------------------------------------------------------------
- * Parse a stevedore argument on the form:
- *	[ name '=' ] strategy [ ',' arg ] *
- */
-
-static const struct choice STV_choice[] = {
-	{ "file",			&smf_stevedore },
-	{ "malloc",			&sma_stevedore },
-	{ "deprecated_persistent",	&smp_stevedore },
-	{ "persistent",			&smp_fake_stevedore },
-#ifdef HAVE_LIBUMEM
-	{ "umem",			&smu_stevedore },
-#endif
-	{ NULL,		NULL }
-};
-
-void
-STV_Config(const char *spec)
-{
-	char **av;
-	const char *p, *q;
-	struct stevedore *stv;
-	const struct stevedore *stv2;
-	int ac, l;
-	static unsigned seq = 0;
-
-	ASSERT_MGT();
-	p = strchr(spec, '=');
-	q = strchr(spec, ',');
-	if (p != NULL && (q == NULL || q > p)) {
-		av = VAV_Parse(p + 1, NULL, ARGV_COMMA);
-	} else {
-		av = VAV_Parse(spec, NULL, ARGV_COMMA);
-		p = NULL;
-	}
-	AN(av);
-
-	if (av[0] != NULL)
-		ARGV_ERR("%s\n", av[0]);
-
-	if (av[1] == NULL)
-		ARGV_ERR("-s argument lacks strategy {malloc, file, ...}\n");
-
-	for (ac = 0; av[ac + 2] != NULL; ac++)
-		continue;
-
-	stv2 = pick(STV_choice, av[1], "storage");
-	AN(stv2);
-
-	/* Append strategy to ident string */
-	VSB_printf(vident, ",-s%s", av[1]);
-
-	av += 2;
-
-	CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC);
-	ALLOC_OBJ(stv, STEVEDORE_MAGIC);
-	AN(stv);
-
-	*stv = *stv2;
-	AN(stv->name);
-
-	if (p == NULL)
-		bprintf(stv->ident, "s%u", seq++);
-	else {
-		l = p - spec;
-		if (l > sizeof stv->ident - 1)
-			l = sizeof stv->ident - 1;
-		bprintf(stv->ident, "%.*s", l, spec);
-	}
-
-	VTAILQ_FOREACH(stv2, &stv_stevedores, list) {
-		if (strcmp(stv2->ident, stv->ident))
-			continue;
-		ARGV_ERR("(-s%s=%s) already defined once\n",
-		    stv->ident, stv->name);
-	}
-
-	if (stv->init != NULL)
-		stv->init(stv, ac, av);
-	else if (ac != 0)
-		ARGV_ERR("(-s%s) too many arguments\n", stv->name);
-
-	AN(stv->alloc);
-	if (stv->allocobj == NULL)
-		stv->allocobj = stv_default_allocobj;
-
-	if (!strcmp(stv->ident, TRANSIENT_STORAGE)) {
-		stv->transient = 1;
-		AZ(stv_transient);
-		stv_transient = stv;
-	} else {
-		VTAILQ_INSERT_TAIL(&stv_stevedores, stv, list);
-	}
-	/* NB: Do not free av, stevedore gets to keep it */
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-STV_Config_Transient(void)
-{
-
-	ASSERT_MGT();
-
-	if (stv_transient == NULL)
-		STV_Config(TRANSIENT_STORAGE "=malloc");
-}
-
-/*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/storage/storage_persistent_mgt.c b/bin/varnishd/storage/storage_persistent_mgt.c
deleted file mode 100644
index 76d6436..0000000
--- a/bin/varnishd/storage/storage_persistent_mgt.c
+++ /dev/null
@@ -1,209 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Persistent storage method
- *
- * XXX: Before we start the client or maybe after it stops, we should give the
- * XXX: stevedores a chance to examine their storage for consistency.
- *
- * XXX: Do we ever free the LRU-lists ?
- */
-
-#include "config.h"
-
-#include <sys/mman.h>
-
-#include <errno.h>
-#include <math.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache/cache.h"
-#include "storage/storage.h"
-
-#include "vsha256.h"
-
-#include "storage/storage_persistent.h"
-
-#ifndef MAP_NOCORE
-#define MAP_NOCORE 0 /* XXX Linux */
-#endif
-
-#ifndef MAP_NOSYNC
-#define MAP_NOSYNC 0 /* XXX Linux */
-#endif
-
-/*--------------------------------------------------------------------
- * Calculate cleaner metrics from silo dimensions
- */
-
-static void
-smp_metrics(struct smp_sc *sc)
-{
-
-	/*
-	 * We do not want to loose too big chunks of the silos
-	 * content when we are forced to clean a segment.
-	 *
-	 * For now insist that a segment covers no more than 1% of the silo.
-	 *
-	 * XXX: This should possibly depend on the size of the silo so
-	 * XXX: trivially small silos do not run into trouble along
-	 * XXX: the lines of "one object per segment".
-	 */
-
-	sc->min_nseg = 10;
-	sc->max_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->min_nseg;
-
-	fprintf(stderr, "min_nseg = %u, max_segl = %ju\n",
-	    sc->min_nseg, (uintmax_t)sc->max_segl);
-
-	/*
-	 * The number of segments are limited by the size of the segment
-	 * table(s) and from that follows the minimum size of a segmement.
-	 */
-
-	sc->max_nseg = smp_stuff_len(sc, SMP_SEG1_STUFF) / sc->min_nseg;
-	sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
-
-	while (sc->min_segl < sizeof(struct object)) {
-		sc->max_nseg /= 2;
-		sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
-	}
-
-	fprintf(stderr, "max_nseg = %u, min_segl = %ju\n",
-	    sc->max_nseg, (uintmax_t)sc->min_segl);
-
-	/*
-	 * Set our initial aim point at the exponential average of the
-	 * two extremes.
-	 *
-	 * XXX: This is a pretty arbitrary choice, but having no idea
-	 * XXX: object count, size distribution or ttl pattern at this
-	 * XXX: point, we have to do something.
-	 */
-
-	sc->aim_nseg =
-	   (unsigned) exp((log(sc->min_nseg) + log(sc->max_nseg))*.5);
-	sc->aim_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->aim_nseg;
-
-	fprintf(stderr, "aim_nseg = %u, aim_segl = %ju\n",
-	    sc->aim_nseg, (uintmax_t)sc->aim_segl);
-
-	/*
-	 * How much space in the free reserve pool ?
-	 */
-	sc->free_reserve = sc->aim_segl * 10;
-
-	fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
-}
-
-/*--------------------------------------------------------------------
- * Set up persistent storage silo in the master process.
- */
-
-void
-smp_mgt_init(struct stevedore *parent, int ac, char * const *av)
-{
-	struct smp_sc		*sc;
-	struct smp_sign		sgn;
-	void *target;
-	int i;
-
-	ASSERT_MGT();
-
-	AZ(av[ac]);
-
-	/* Necessary alignment. See also smp_object::__filler__ */
-	assert(sizeof(struct smp_object) % 8 == 0);
-
-#define SIZOF(foo)       fprintf(stderr, \
-    "sizeof(%s) = %zu = 0x%zx\n", #foo, sizeof(foo), sizeof(foo));
-	SIZOF(struct smp_ident);
-	SIZOF(struct smp_sign);
-	SIZOF(struct smp_segptr);
-	SIZOF(struct smp_object);
-#undef SIZOF
-
-	/* See comments in storage_persistent.h */
-	assert(sizeof(struct smp_ident) == SMP_IDENT_SIZE);
-
-	/* Allocate softc */
-	ALLOC_OBJ(sc, SMP_SC_MAGIC);
-	XXXAN(sc);
-	sc->parent = parent;
-	sc->fd = -1;
-	VTAILQ_INIT(&sc->segments);
-
-	/* Argument processing */
-	if (ac != 2)
-		ARGV_ERR("(-spersistent) wrong number of arguments\n");
-
-	i = STV_GetFile(av[0], &sc->fd, &sc->filename, "-spersistent");
-	if (i == 2)
-		ARGV_ERR("(-spersistent) need filename (not directory)\n");
-
-	sc->align = sizeof(void*) * 2;
-	sc->granularity = getpagesize();
-	sc->mediasize = STV_FileSize(sc->fd, av[1], &sc->granularity,
-	    "-spersistent");
-
-	AZ(ftruncate(sc->fd, sc->mediasize));
-
-	/* Try to determine correct mmap address */
-	i = read(sc->fd, &sgn, sizeof sgn);
-	assert(i == sizeof sgn);
-	if (!strcmp(sgn.ident, "SILO"))
-		target = (void*)(uintptr_t)sgn.mapped;
-	else
-		target = NULL;
-
-	sc->base = (void*)mmap(target, sc->mediasize, PROT_READ|PROT_WRITE,
-	    MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, 0);
-
-	if (sc->base == MAP_FAILED)
-		ARGV_ERR("(-spersistent) failed to mmap (%s)\n",
-		    strerror(errno));
-
-	smp_def_sign(sc, &sc->idn, 0, "SILO");
-	sc->ident = SIGN_DATA(&sc->idn);
-
-	i = smp_valid_silo(sc);
-	if (i) {
-		printf("Warning SILO (%s) not reloaded (reason=%d)\n",
-		    sc->filename, i);
-		smp_newsilo(sc);
-	}
-	AZ(smp_valid_silo(sc));
-
-	smp_metrics(sc);
-
-	parent->priv = sc;
-
-	/* XXX: only for sendfile I guess... */
-	mgt_child_inherit(sc->fd, "storage_persistent");
-}



From phk at FreeBSD.org  Tue May 26 09:34:13 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 26 May 2015 11:34:13 +0200
Subject: [master] 0d9759c Introduce the struct "waitfor" so that the same
 waiter can be used for more than one kind of waiting.
Message-ID: <E1YxBFV-0005LN-24@project.varnish-software.com>


commit 0d9759c8a257567e8115fc62cb8466c5f0cf736f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 26 09:33:46 2015 +0000

    Introduce the struct "waitfor" so that the same waiter can be used
    for more than one kind of waiting.

diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index 16b989e..f130abd 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -55,6 +55,7 @@ struct tcp_pool {
 	int			refcnt;
 	struct lock		mtx;
 
+	struct waitfor		waitfor;
 	struct waiter		*waiter;
 	volatile double		timeout;
 
@@ -162,7 +163,10 @@ VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6)
 	VTAILQ_INIT(&tp->killlist);
 	VTAILQ_INSERT_HEAD(&pools, tp, list);
 	tp->timeout = 60;
-	tp->waiter = Waiter_New(tcp_handle, &tp->timeout);
+	INIT_OBJ(&tp->waitfor, WAITFOR_MAGIC);
+	tp->waitfor.func = tcp_handle;
+	tp->waitfor.tmo = &tp->timeout;
+	tp->waiter = Waiter_New(&tp->waitfor);
 	return (tp);
 }
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 950023c..d49decc 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -573,6 +573,8 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
  * Create and delete pools
  */
 
+static struct waitfor ses_wf;
+
 struct sesspool *
 SES_NewPool(struct pool *wp, unsigned pool_no)
 {
@@ -588,7 +590,11 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	bprintf(nb, "sess%u", pool_no);
 	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool,
 	    &cache_param->workspace_session);
-	pp->http1_waiter = Waiter_New(ses_handle, &cache_param->timeout_idle);
+
+	INIT_OBJ(&ses_wf, WAITFOR_MAGIC);
+	ses_wf.func = ses_handle;
+	ses_wf.tmo = &cache_param->timeout_idle;
+	pp->http1_waiter = Waiter_New(&ses_wf);
 
 	VCA_New_SessPool(wp, pp);
 	return (pp);
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index 5e76eb0..46957be 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -62,7 +62,7 @@ Waiter_GetName(void)
 }
 
 struct waiter *
-Waiter_New(waiter_handle_f *func, volatile double *tmo)
+Waiter_New(struct waitfor *wf)
 {
 	struct waiter *w;
 
@@ -77,8 +77,7 @@ Waiter_New(waiter_handle_f *func, volatile double *tmo)
 	INIT_OBJ(w, WAITER_MAGIC);
 	w->priv = (void*)(w + 1);
 	w->impl = waiter;
-	w->func = func;
-	w->tmo = tmo;
+	w->waitfor = wf;
 	VTAILQ_INIT(&w->waithead);
 
 	waiter->init(w);
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index 9302add..bba3754 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -71,13 +71,13 @@ vwe_eev(struct vwe *vwe, const struct epoll_event *ep, double now)
 	AN(ep->data.ptr);
 	CAST_OBJ_NOTNULL(wp, ep->data.ptr, WAITED_MAGIC);
 	if (ep->events & EPOLLIN) {
-		vwe->waiter->func(wp, WAITER_ACTION, now);
+		Wait_Call(vwe->waiter, wp, WAITER_ACTION, now);
 	} else if (ep->events & EPOLLERR) {
-		vwe->waiter->func(wp, WAITER_REMCLOSE, now);
+		Wait_Call(vwe->waiter, wp, WAITER_REMCLOSE, now);
 	} else if (ep->events & EPOLLHUP) {
-		vwe->waiter->func(wp, WAITER_REMCLOSE, now);
+		Wait_Call(vwe->waiter, wp, WAITER_REMCLOSE, now);
 	} else if (ep->events & EPOLLRDHUP) {
-		vwe->waiter->func(wp, WAITER_REMCLOSE, now);
+		Wait_Call(vwe->waiter, wp, WAITER_REMCLOSE, now);
 	}
 }
 
@@ -99,7 +99,7 @@ vwe_thread(void *priv)
 
 	last_idle = 0.0;
 	while (1) {
-		i = floor(.3 * 1e3 * *vwe->waiter->tmo);
+		i = floor(.3 * 1e3 * Wait_Tmo(vwe->waiter, NULL));
 		n = epoll_wait(vwe->epfd, ev, NEEV, i);
 		if (n < 0 && vwe->die)
 			break;
@@ -113,8 +113,8 @@ vwe_thread(void *priv)
 			Lck_Unlock(&vwe->mtx);
 			vwe_eev(vwe, ep, now);
 		}
-		idle = now - *vwe->waiter->tmo;
-		if (now - last_idle < .3 * *vwe->waiter->tmo)
+		idle = now - Wait_Tmo(vwe->waiter, NULL);
+		if (now - last_idle < .3 * Wait_Tmo(vwe->waiter, NULL))
 			continue;
 		last_idle = now;
 		VTAILQ_INIT(&tlist);
@@ -132,7 +132,7 @@ vwe_thread(void *priv)
 			if (wp == NULL)
 				break;
 			VTAILQ_REMOVE(&tlist, wp, list);
-			vwe->waiter->func(wp, WAITER_TIMEOUT, now);
+			Wait_Call(vwe->waiter, wp, WAITER_TIMEOUT, now);
 		}
 	}
 	return (NULL);
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index 6c55ac9..a6d4c8c 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -75,7 +75,7 @@ vwk_thread(void *priv)
 
 	last_idle = 0.0;
 	while (1) {
-		now = .3 * *vwk->waiter->tmo;
+		now = .3 * Wait_Tmo(vwk->waiter, NULL);
 		ts.tv_sec = (time_t)floor(now);
 		ts.tv_nsec = (long)(1e9 * (now - ts.tv_sec));
 		n = kevent(vwk->kq, NULL, 0, ke, NKEV, &ts);
@@ -91,12 +91,12 @@ vwk_thread(void *priv)
 			VTAILQ_REMOVE(&vwk->list, wp, list);
 			Lck_Unlock(&vwk->mtx);
 			if (kp->flags & EV_EOF)
-				vwk->waiter->func(wp, WAITER_REMCLOSE, now);
+				Wait_Call(vwk->waiter, wp, WAITER_REMCLOSE, now);
 			else
-				vwk->waiter->func(wp, WAITER_ACTION, now);
+				Wait_Call(vwk->waiter, wp, WAITER_ACTION, now);
 		}
-		idle = now - *vwk->waiter->tmo;
-		if (now - last_idle < .3 * *vwk->waiter->tmo)
+		idle = now - Wait_Tmo(vwk->waiter, NULL);
+		if (now - last_idle < .3 * Wait_Tmo(vwk->waiter, NULL))
 			continue;
 		last_idle = now;
 		n = 0;
@@ -114,7 +114,7 @@ vwk_thread(void *priv)
 		for (j = 0; j < n; j++) {
 			CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC);
 			VTAILQ_REMOVE(&vwk->list, wp, list);
-			vwk->waiter->func(wp, WAITER_TIMEOUT, now);
+			Wait_Call(vwk->waiter, wp, WAITER_TIMEOUT, now);
 		}
 		Lck_Unlock(&vwk->mtx);
 	}
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index fba970b..2de4402 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -162,12 +162,12 @@ vwp_main(void *priv)
 
 	while (1) {
 		v = poll(vwp->pollfd, vwp->hpoll,
-		    (int)floor(1e3 * *vwp->waiter->tmo));
+		    (int)floor(1e3 * Wait_Tmo(vwp->waiter, NULL)));
 		assert(v >= 0);
 		if (v == 0)
 			v = vwp->hpoll;
 		now = VTIM_real();
-		idle = now - *vwp->waiter->tmo;
+		idle = now - Wait_Tmo(vwp->waiter, NULL);
 		i = 0;
 		dopipe = 0;
 		while (v > 0 && i < vwp->hpoll) {
@@ -182,12 +182,12 @@ vwp_main(void *priv)
 			wp = vwp->idx[i];
 			CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 			if (wp->idle <= idle) {
-				vwp->waiter->func(wp, WAITER_TIMEOUT, now);
+				Wait_Call(vwp->waiter, wp, WAITER_TIMEOUT, now);
 				vwp_del(vwp, i);
 			} else if (vwp->pollfd[i].revents & POLLIN) {
 				assert(wp->fd > 0);
 				assert(wp->fd == vwp->pollfd[i].fd);
-				vwp->waiter->func(wp, WAITER_ACTION, now);
+				Wait_Call(vwp->waiter, wp, WAITER_ACTION, now);
 				vwp_del(vwp, i);
 			} else {
 				i++;
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
index 500b36a..0e70d9e 100644
--- a/bin/varnishd/waiter/waiter.h
+++ b/bin/varnishd/waiter/waiter.h
@@ -42,11 +42,24 @@
  * Public interfaces
  */
 
+struct waited;
 struct waiter;
 
-/* Connection waiter -------------------------------------------------
- * Describing a file-descriptor/connection being waited on
- */
+enum wait_event {
+	WAITER_REMCLOSE,
+	WAITER_TIMEOUT,
+	WAITER_ACTION,
+	WAITER_CLOSE
+};
+
+typedef void waiter_handle_f(struct waited *, enum wait_event, double now);
+
+struct waitfor {
+	unsigned		magic;
+#define WAITFOR_MAGIC		0x16b79246
+	waiter_handle_f		*func;
+	volatile double		*tmo;
+};
 
 struct waited {
 	unsigned		magic;
@@ -57,19 +70,8 @@ struct waited {
 	VTAILQ_ENTRY(waited)	list;
 };
 
-enum wait_event {
-	WAITER_REMCLOSE,
-	WAITER_TIMEOUT,
-	WAITER_ACTION,
-	WAITER_CLOSE
-};
-
-#define WAITER_DEFAULT		"platform dependent"
-
-typedef void waiter_handle_f(struct waited *, enum wait_event, double now);
-
 /* cache_waiter.c */
 int Wait_Enter(const struct waiter *, struct waited *);
-struct waiter *Waiter_New(waiter_handle_f *, volatile double *timeout);
+struct waiter *Waiter_New(struct waitfor *);
 void Waiter_Destroy(struct waiter **);
 const char *Waiter_GetName(void);
diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h
index 61ff970..5890912 100644
--- a/bin/varnishd/waiter/waiter_priv.h
+++ b/bin/varnishd/waiter/waiter_priv.h
@@ -40,12 +40,10 @@ struct waiter {
 
 	int				dismantle;
 
-	waiter_handle_f *		func;
+	struct waitfor			*waitfor;
 
 	double				next_idle;
 
-	volatile double			*tmo;
-
 	void				*priv;
 };
 
@@ -56,10 +54,31 @@ typedef void waiter_inject_f(const struct waiter *, struct waited *);
 typedef void waiter_evict_f(const struct waiter *, struct waited *);
 
 struct waiter_impl {
-	const char		*name;
-	waiter_init_f		*init;
-	waiter_fini_f		*fini;
-	waiter_enter_f		*enter;
-	waiter_inject_f		*inject;
-	size_t			size;
+	const char			*name;
+	waiter_init_f			*init;
+	waiter_fini_f			*fini;
+	waiter_enter_f			*enter;
+	waiter_inject_f			*inject;
+	size_t				size;
 };
+
+static inline double
+Wait_Tmo(const struct waiter *w, const struct waited *wp)
+{
+	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
+	CHECK_OBJ_ORNULL(wp, WAITED_MAGIC);
+	CHECK_OBJ_NOTNULL(w->waitfor, WAITFOR_MAGIC);
+	AN(w->waitfor->tmo);
+	return (*w->waitfor->tmo);
+}
+
+static inline void
+Wait_Call(const struct waiter *w,
+    struct waited *wp, enum wait_event ev, double now)
+{
+	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
+	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
+	CHECK_OBJ_NOTNULL(w->waitfor, WAITFOR_MAGIC);
+	AN(w->waitfor->func);
+	w->waitfor->func(wp, ev, now);
+}



From tfheen at fastly.com  Tue May 26 09:54:41 2015
From: tfheen at fastly.com (Tollef Fog Heen)
Date: Tue, 26 May 2015 11:54:41 +0200
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <35990.1432465498@critter.freebsd.dk>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
 <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>
 <555E5690.8090706@schokola.de>
 <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>
 <555F2F8D.9030402@uplex.de>
 <CAJV_h0bvYpJ3DuQ+-z-jP_f+wkkOQyLY9pjdmP4bs+WyjSA9rw@mail.gmail.com>
 <5561A55B.4040608@uplex.de> <35990.1432465498@critter.freebsd.dk>
Message-ID: <CA+nLBF0hNwLoTbEpQ=aP27KbiVbV6f-Uuc3Ffh-_e-2Q5_-1yg@mail.gmail.com>

2015-05-24 13:04 GMT+02:00 Poul-Henning Kamp <phk at phk.freebsd.dk>:

> The fact that *building* a VMOD requires python is not a good reason
> to make the Varnish *run-time* package depend on python.

Surely vmodtool.py goes into the varnish development package, which
does depend on Python.

-- 
Tollef Fog Heen | Operations Engineer
fastly.com | @fastly | Linkedin



From dridi at varni.sh  Tue May 26 10:02:43 2015
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Tue, 26 May 2015 12:02:43 +0200
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <CA+nLBF0hNwLoTbEpQ=aP27KbiVbV6f-Uuc3Ffh-_e-2Q5_-1yg@mail.gmail.com>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
 <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>
 <555E5690.8090706@schokola.de>
 <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>
 <555F2F8D.9030402@uplex.de>
 <CAJV_h0bvYpJ3DuQ+-z-jP_f+wkkOQyLY9pjdmP4bs+WyjSA9rw@mail.gmail.com>
 <5561A55B.4040608@uplex.de> <35990.1432465498@critter.freebsd.dk>
 <CA+nLBF0hNwLoTbEpQ=aP27KbiVbV6f-Uuc3Ffh-_e-2Q5_-1yg@mail.gmail.com>
Message-ID: <CABoVN9AO-f6n0X=_TEaZQMZi61JPVhSDB47XiHthVpuMiavEmQ@mail.gmail.com>

On Tue, May 26, 2015 at 11:54 AM, Tollef Fog Heen <tfheen at fastly.com> wrote:
> 2015-05-24 13:04 GMT+02:00 Poul-Henning Kamp <phk at phk.freebsd.dk>:
>
>> The fact that *building* a VMOD requires python is not a good reason
>> to make the Varnish *run-time* package depend on python.
>
> Surely vmodtool.py goes into the varnish development package, which
> does depend on Python.

Not on Fedora, so probably not on EL distros.

$ rpm -qR varnish-libs-devel
/usr/bin/env
/usr/bin/pkg-config
rpmlib(CompressedFileNames) <= 3.0.4-1
rpmlib(FileDigests) <= 4.6.0-1
rpmlib(PayloadFilesHavePrefix) <= 4.0-1
rpmlib(PayloadIsXz) <= 5.2-1
varnish-libs = 4.0.3-3.fc21

$ rpm -ql varnish-libs-devel | grep py
/usr/share/varnish/vmodtool.py
/usr/share/varnish/vmodtool.pyc
/usr/share/varnish/vmodtool.pyo

Dridi

> --
> Tollef Fog Heen | Operations Engineer
> fastly.com | @fastly | Linkedin
>
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit



From fgsch at lodoss.net  Tue May 26 20:31:31 2015
From: fgsch at lodoss.net (Federico Schwindt)
Date: Tue, 26 May 2015 21:31:31 +0100
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <CABoVN9AO-f6n0X=_TEaZQMZi61JPVhSDB47XiHthVpuMiavEmQ@mail.gmail.com>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
 <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>
 <555E5690.8090706@schokola.de>
 <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>
 <555F2F8D.9030402@uplex.de>
 <CAJV_h0bvYpJ3DuQ+-z-jP_f+wkkOQyLY9pjdmP4bs+WyjSA9rw@mail.gmail.com>
 <5561A55B.4040608@uplex.de> <35990.1432465498@critter.freebsd.dk>
 <CA+nLBF0hNwLoTbEpQ=aP27KbiVbV6f-Uuc3Ffh-_e-2Q5_-1yg@mail.gmail.com>
 <CABoVN9AO-f6n0X=_TEaZQMZi61JPVhSDB47XiHthVpuMiavEmQ@mail.gmail.com>
Message-ID: <CAJV_h0YL91CVT2pKkdkxDyMpn8xx6MAwMicR1+SkPnoPXYp2=Q@mail.gmail.com>

I'd say that's a bug.

On Tue, May 26, 2015 at 11:02 AM, Dridi Boukelmoune <dridi at varni.sh> wrote:

> On Tue, May 26, 2015 at 11:54 AM, Tollef Fog Heen <tfheen at fastly.com>
> wrote:
> > 2015-05-24 13:04 GMT+02:00 Poul-Henning Kamp <phk at phk.freebsd.dk>:
> >
> >> The fact that *building* a VMOD requires python is not a good reason
> >> to make the Varnish *run-time* package depend on python.
> >
> > Surely vmodtool.py goes into the varnish development package, which
> > does depend on Python.
>
> Not on Fedora, so probably not on EL distros.
>
> $ rpm -qR varnish-libs-devel
> /usr/bin/env
> /usr/bin/pkg-config
> rpmlib(CompressedFileNames) <= 3.0.4-1
> rpmlib(FileDigests) <= 4.6.0-1
> rpmlib(PayloadFilesHavePrefix) <= 4.0-1
> rpmlib(PayloadIsXz) <= 5.2-1
> varnish-libs = 4.0.3-3.fc21
>
> $ rpm -ql varnish-libs-devel | grep py
> /usr/share/varnish/vmodtool.py
> /usr/share/varnish/vmodtool.pyc
> /usr/share/varnish/vmodtool.pyo
>
> Dridi
>
> > --
> > Tollef Fog Heen | Operations Engineer
> > fastly.com | @fastly | Linkedin
> >
> > _______________________________________________
> > varnish-commit mailing list
> > varnish-commit at varnish-cache.org
> > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit
>
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/vinyl-commit/attachments/20150526/d600734c/attachment.html>

From phk at FreeBSD.org  Tue May 26 20:36:43 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 26 May 2015 22:36:43 +0200
Subject: [master] 68cf6f8 Split 'struct pool' into a private include file.
Message-ID: <E1YxLad-00047z-TF@project.varnish-software.com>


commit 68cf6f868c0d8ae63914839b583467d2369620f2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 26 19:29:47 2015 +0000

    Split 'struct pool' into a private include file.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index ac3b216..1b11581 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -100,6 +100,7 @@ varnishd_SOURCES = \
 noinst_HEADERS = \
 	builtin_vcl.h \
 	cache/cache_esi.h \
+	cache/cache_pool.h \
 	common/heritage.h \
 	hash/hash_slinger.h \
 	mgt/mgt.h \
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 4bec380..730966c 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -39,35 +39,10 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "cache_pool.h"
 
 #include "vtim.h"
 
-VTAILQ_HEAD(taskhead, pool_task);
-
-/* Number of work requests queued in excess of worker threads available */
-
-struct pool {
-	unsigned			magic;
-#define POOL_MAGIC			0x606658fa
-	VTAILQ_ENTRY(pool)		list;
-
-	pthread_cond_t			herder_cond;
-	pthread_t			herder_thr;
-
-	struct lock			mtx;
-	struct taskhead			idle_queue;
-	struct taskhead			front_queue;
-	struct taskhead			back_queue;
-	unsigned			nthr;
-	unsigned			dry;
-	unsigned			lqueue;
-	uintmax_t			ndropped;
-	uintmax_t			nqueued;
-	struct sesspool			*sesspool;
-	struct dstat			*a_stat;
-	struct dstat			*b_stat;
-};
-
 static struct lock		pool_mtx;
 static pthread_t		thr_pool_herder;
 
diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
new file mode 100644
index 0000000..7660d7a
--- /dev/null
+++ b/bin/varnishd/cache/cache_pool.h
@@ -0,0 +1,56 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Private include file for the pool aware code.
+ */
+
+#include "config.h"
+
+VTAILQ_HEAD(taskhead, pool_task);
+
+struct pool {
+	unsigned			magic;
+#define POOL_MAGIC			0x606658fa
+	VTAILQ_ENTRY(pool)		list;
+
+	pthread_cond_t			herder_cond;
+	pthread_t			herder_thr;
+
+	struct lock			mtx;
+	struct taskhead			idle_queue;
+	struct taskhead			front_queue;
+	struct taskhead			back_queue;
+	unsigned			nthr;
+	unsigned			dry;
+	unsigned			lqueue;
+	uintmax_t			ndropped;
+	uintmax_t			nqueued;
+	struct sesspool			*sesspool;
+	struct dstat			*a_stat;
+	struct dstat			*b_stat;
+};



From phk at FreeBSD.org  Tue May 26 20:36:43 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 26 May 2015 22:36:43 +0200
Subject: [master] f71729a Move pool stuff related to the workthreads of each
 pool to cache_wrk.c,
 but retain stats stuff (since it runs across pools) in cache_pool.c
Message-ID: <E1YxLad-000485-UT@project.varnish-software.com>


commit f71729aec2aaacf280ddeafe7891f4103393cd14
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 26 19:59:42 2015 +0000

    Move pool stuff related to the workthreads of each pool to cache_wrk.c,
    but retain stats stuff (since it runs across pools) in cache_pool.c

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ba45bf2..bb3c61d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -962,7 +962,6 @@ const char *sess_close_2str(enum sess_close sc, int want_desc);
 
 /* cache_pool.c */
 void Pool_Init(void);
-void Pool_Work_Thread(struct pool *, struct worker *w);
 int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how);
 int Pool_Task_Arg(struct worker *, task_func_t *,
     const void *arg, size_t arg_len);
@@ -1110,7 +1109,6 @@ void VMOD_Init(void);
 
 /* cache_wrk.c */
 
-void WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace);
 typedef void *bgthread_t(struct worker *, void *priv);
 void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
     void *priv);
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 730966c..f71980d 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -41,12 +41,10 @@
 #include "cache.h"
 #include "cache_pool.h"
 
-#include "vtim.h"
-
-static struct lock		pool_mtx;
 static pthread_t		thr_pool_herder;
 
 static struct lock		wstat_mtx;
+struct lock			pool_mtx;
 
 /*--------------------------------------------------------------------
  * Summing of stats into global stats counters
@@ -88,25 +86,6 @@ Pool_TrySumstat(struct worker *wrk)
 }
 
 /*--------------------------------------------------------------------
- * Summing of stats into pool counters
- */
-
-static void
-pool_addstat(struct dstat *dst, struct dstat *src)
-{
-
-	dst->summs++;
-#define L0(n)
-#define L1(n) (dst->n += src->n)
-#define VSC_F(n,t,l,s,f,v,d,e)	L##l(n);
-#include "tbl/vsc_f_main.h"
-#undef VSC_F
-#undef L0
-#undef L1
-	memset(src, 0, sizeof *src);
-}
-
-/*--------------------------------------------------------------------
  * Helper function to update stats for purges under lock
  */
 
@@ -119,145 +98,11 @@ Pool_PurgeStat(unsigned nobj)
 	Lck_Unlock(&wstat_mtx);
 }
 
-
-/*--------------------------------------------------------------------
- */
-
-static struct worker *
-pool_getidleworker(struct pool *pp)
-{
-	struct pool_task *pt;
-	struct worker *wrk;
-
-	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
-	Lck_AssertHeld(&pp->mtx);
-	pt = VTAILQ_FIRST(&pp->idle_queue);
-	if (pt == NULL) {
-		if (pp->nthr < cache_param->wthread_max) {
-			pp->dry++;
-			AZ(pthread_cond_signal(&pp->herder_cond));
-		}
-		return (NULL);
-	}
-	AZ(pt->func);
-	CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
-	return (wrk);
-}
-
-/*--------------------------------------------------------------------
- * Special scheduling:  If no thread can be found, the current thread
- * will be prepared for rescheduling instead.
- * The selected threads workspace is reserved and the argument put there.
- * Return one if another thread was scheduled, otherwise zero.
- */
-
-int
-Pool_Task_Arg(struct worker *wrk, task_func_t *func,
-    const void *arg, size_t arg_len)
-{
-	struct pool *pp;
-	struct worker *wrk2;
-	int retval;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	AN(arg);
-	AN(arg_len);
-	pp = wrk->pool;
-	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
-
-	Lck_Lock(&pp->mtx);
-	wrk2 = pool_getidleworker(pp);
-	if (wrk2 != NULL) {
-		VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list);
-		retval = 1;
-	} else {
-		wrk2 = wrk;
-		retval = 0;
-	}
-	Lck_Unlock(&pp->mtx);
-	AZ(wrk2->task.func);
-
-	assert(arg_len <= WS_Reserve(wrk2->aws, arg_len));
-	memcpy(wrk2->aws->f, arg, arg_len);
-	wrk2->task.func = func;
-	wrk2->task.priv = wrk2->aws->f;
-	if (retval)
-		AZ(pthread_cond_signal(&wrk2->cond));
-	return (retval);
-}
-
-/*--------------------------------------------------------------------
- * Enter a new task to be done
- */
-
-int
-Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
-{
-	struct worker *wrk;
-	int retval = 0;
-
-	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
-	AN(task);
-	AN(task->func);
-
-	Lck_Lock(&pp->mtx);
-
-	/*
-	 * The common case first:  Take an idle thread, do it.
-	 */
-
-	wrk = pool_getidleworker(pp);
-	if (wrk != NULL) {
-		VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
-		AZ(wrk->task.func);
-		wrk->task.func = task->func;
-		wrk->task.priv = task->priv;
-		Lck_Unlock(&pp->mtx);
-		AZ(pthread_cond_signal(&wrk->cond));
-		return (0);
-	}
-
-	switch (how) {
-	case POOL_NO_QUEUE:
-		retval = -1;
-		break;
-	case POOL_QUEUE_FRONT:
-		/* If we have too much in the queue already, refuse. */
-		if (pp->lqueue > cache_param->wthread_queue_limit) {
-			pp->ndropped++;
-			retval = -1;
-		} else {
-			VTAILQ_INSERT_TAIL(&pp->front_queue, task, list);
-			pp->nqueued++;
-			pp->lqueue++;
-		}
-		break;
-	case POOL_QUEUE_BACK:
-		VTAILQ_INSERT_TAIL(&pp->back_queue, task, list);
-		break;
-	default:
-		WRONG("Unknown enum pool_how");
-	}
-	Lck_Unlock(&pp->mtx);
-	return (retval);
-}
-
-/*--------------------------------------------------------------------
- * Empty function used as a pointer value for the thread exit condition.
- */
-
-static void __match_proto__(task_func_t)
-pool_kiss_of_death(struct worker *wrk, void *priv)
-{
-	(void)wrk;
-	(void)priv;
-}
-
 /*--------------------------------------------------------------------
  * Special function to summ stats
  */
 
-static void __match_proto__(task_func_t)
+void __match_proto__(task_func_t)
 pool_stat_summ(struct worker *wrk, void *priv)
 {
 	struct dstat *src;
@@ -274,235 +119,6 @@ pool_stat_summ(struct worker *wrk, void *priv)
 }
 
 /*--------------------------------------------------------------------
- * This is the work function for worker threads in the pool.
- */
-
-void
-Pool_Work_Thread(struct pool *pp, struct worker *wrk)
-{
-	struct pool_task *tp;
-	struct pool_task tpx, tps;
-	int i;
-
-	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
-	wrk->pool = pp;
-	while (1) {
-		Lck_Lock(&pp->mtx);
-
-		CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-
-		WS_Reset(wrk->aws, NULL);
-		AZ(wrk->vsl);
-
-		tp = VTAILQ_FIRST(&pp->front_queue);
-		if (tp != NULL) {
-			pp->lqueue--;
-			VTAILQ_REMOVE(&pp->front_queue, tp, list);
-		} else {
-			tp = VTAILQ_FIRST(&pp->back_queue);
-			if (tp != NULL)
-				VTAILQ_REMOVE(&pp->back_queue, tp, list);
-		}
-
-		if ((tp == NULL && wrk->stats->summs > 0) ||
-		    (wrk->stats->summs >= cache_param->wthread_stats_rate))
-			pool_addstat(pp->a_stat, wrk->stats);
-
-		if (tp != NULL) {
-			wrk->stats->summs++;
-		} else if (pp->b_stat != NULL && pp->a_stat->summs) {
-			/* Nothing to do, push pool stats into global pool */
-			tps.func = pool_stat_summ;
-			tps.priv = pp->a_stat;
-			pp->a_stat = pp->b_stat;
-			pp->b_stat = NULL;
-			tp = &tps;
-		} else {
-			/* Nothing to do: To sleep, perchance to dream ... */
-			if (isnan(wrk->lastused))
-				wrk->lastused = VTIM_real();
-			wrk->task.func = NULL;
-			wrk->task.priv = wrk;
-			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
-			do {
-				i = Lck_CondWait(&wrk->cond, &pp->mtx,
-				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
-				if (i == ETIMEDOUT)
-					VCL_Rel(&wrk->vcl);
-			} while (wrk->task.func == NULL);
-			tpx = wrk->task;
-			tp = &tpx;
-			wrk->stats->summs++;
-		}
-		Lck_Unlock(&pp->mtx);
-
-		if (tp->func == pool_kiss_of_death)
-			break;
-
-		do {
-			memset(&wrk->task, 0, sizeof wrk->task);
-			assert(wrk->pool == pp);
-			tp->func(wrk, tp->priv);
-			tpx = wrk->task;
-			tp = &tpx;
-		} while (tp->func != NULL);
-
-		/* cleanup for next task */
-		wrk->seen_methods = 0;
-	}
-	wrk->pool = NULL;
-}
-
-/*--------------------------------------------------------------------
- * Create another worker thread.
- */
-
-struct pool_info {
-	unsigned		magic;
-#define POOL_INFO_MAGIC		0x4e4442d3
-	size_t			stacksize;
-	struct pool		*qp;
-};
-
-static void *
-pool_thread(void *priv)
-{
-	struct pool_info *pi;
-
-	CAST_OBJ_NOTNULL(pi, priv, POOL_INFO_MAGIC);
-	WRK_Thread(pi->qp, pi->stacksize, cache_param->workspace_thread);
-	FREE_OBJ(pi);
-	return (NULL);
-}
-
-static void
-pool_breed(struct pool *qp)
-{
-	pthread_t tp;
-	pthread_attr_t tp_attr;
-	struct pool_info *pi;
-
-	AZ(pthread_attr_init(&tp_attr));
-	AZ(pthread_attr_setdetachstate(&tp_attr, PTHREAD_CREATE_DETACHED));
-
-	/* Set the stacksize for worker threads we create */
-	if (cache_param->wthread_stacksize != UINT_MAX)
-		AZ(pthread_attr_setstacksize(&tp_attr,
-		    cache_param->wthread_stacksize));
-
-	ALLOC_OBJ(pi, POOL_INFO_MAGIC);
-	AN(pi);
-	AZ(pthread_attr_getstacksize(&tp_attr, &pi->stacksize));
-	pi->qp = qp;
-
-	if (pthread_create(&tp, &tp_attr, pool_thread, pi)) {
-		VSL(SLT_Debug, 0, "Create worker thread failed %d %s",
-		    errno, strerror(errno));
-		Lck_Lock(&pool_mtx);
-		VSC_C_main->threads_failed++;
-		Lck_Unlock(&pool_mtx);
-		VTIM_sleep(cache_param->wthread_fail_delay);
-	} else {
-		qp->dry = 0;
-		qp->nthr++;
-		Lck_Lock(&pool_mtx);
-		VSC_C_main->threads++;
-		VSC_C_main->threads_created++;
-		Lck_Unlock(&pool_mtx);
-		VTIM_sleep(cache_param->wthread_add_delay);
-	}
-
-	AZ(pthread_attr_destroy(&tp_attr));
-}
-
-/*--------------------------------------------------------------------
- * Herd a single pool
- *
- * This thread wakes up whenever a pool queues.
- *
- * The trick here is to not be too aggressive about creating threads.
- * We do this by only examining one pool at a time, and by sleeping
- * a short while whenever we create a thread and a little while longer
- * whenever we fail to, hopefully missing a lot of cond_signals in
- * the meantime.
- *
- * XXX: probably need a lot more work.
- *
- */
-
-static void*
-pool_herder(void *priv)
-{
-	struct pool *pp;
-	struct pool_task *pt;
-	double t_idle;
-	struct worker *wrk;
-
-	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
-
-	while (1) {
-		/* Make more threads if needed and allowed */
-		if (pp->nthr < cache_param->wthread_min ||
-		    (pp->dry && pp->nthr < cache_param->wthread_max)) {
-			pool_breed(pp);
-			continue;
-		}
-		assert(pp->nthr >= cache_param->wthread_min);
-
-		if (pp->nthr > cache_param->wthread_min) {
-
-			t_idle = VTIM_real() - cache_param->wthread_timeout;
-
-			Lck_Lock(&pp->mtx);
-			/* XXX: unsafe counters */
-			VSC_C_main->sess_queued += pp->nqueued;
-			VSC_C_main->sess_dropped += pp->ndropped;
-			pp->nqueued = pp->ndropped = 0;
-
-			wrk = NULL;
-			pt = VTAILQ_LAST(&pp->idle_queue, taskhead);
-			if (pt != NULL) {
-				AZ(pt->func);
-				CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
-
-				if (wrk->lastused < t_idle ||
-				    pp->nthr > cache_param->wthread_max) {
-					/* Give it a kiss on the cheek... */
-					VTAILQ_REMOVE(&pp->idle_queue,
-					    &wrk->task, list);
-					wrk->task.func = pool_kiss_of_death;
-					AZ(pthread_cond_signal(&wrk->cond));
-				} else
-					wrk = NULL;
-			}
-			Lck_Unlock(&pp->mtx);
-
-			if (wrk != NULL) {
-				pp->nthr--;
-				Lck_Lock(&pool_mtx);
-				VSC_C_main->threads--;
-				VSC_C_main->threads_destroyed++;
-				Lck_Unlock(&pool_mtx);
-				VTIM_sleep(cache_param->wthread_destroy_delay);
-				continue;
-			}
-		}
-
-		Lck_Lock(&pp->mtx);
-		if (!pp->dry) {
-			(void)Lck_CondWait(&pp->herder_cond, &pp->mtx,
-				VTIM_real() + 5);
-		} else {
-			/* XXX: unsafe counters */
-			VSC_C_main->threads_limited++;
-			pp->dry = 0;
-		}
-		Lck_Unlock(&pp->mtx);
-	}
-	NEEDLESS_RETURN(NULL);
-}
-
-/*--------------------------------------------------------------------
  * Add a thread pool
  */
 
@@ -538,6 +154,9 @@ pool_mkpool(unsigned pool_no)
 /*--------------------------------------------------------------------
  * This thread adjusts the number of pools to match the parameter.
  *
+ * NB: This is quite silly.  The master should tell the child through
+ * NB: CLI when parameters change and an appropriate call-out table
+ * NB: be maintained for params which require action.
  */
 
 static void *
diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
index 7660d7a..76acc08 100644
--- a/bin/varnishd/cache/cache_pool.h
+++ b/bin/varnishd/cache/cache_pool.h
@@ -54,3 +54,7 @@ struct pool {
 	struct dstat			*a_stat;
 	struct dstat			*b_stat;
 };
+
+void *pool_herder(void*);
+task_func_t pool_stat_summ;
+extern struct lock			pool_mtx;
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index c793176..e95b1eb 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -31,11 +31,18 @@
 
 #include "config.h"
 
+#include <errno.h>
 #include <stdlib.h>
 
 #include "cache.h"
+#include "cache_pool.h"
+
+#include "vtim.h"
 
 #include "hash/hash_slinger.h"
+
+static void Pool_Work_Thread(struct pool *pp, struct worker *wrk);
+
 /*--------------------------------------------------------------------
  * Create and starte a back-ground thread which as its own worker and
  * session data structures;
@@ -82,7 +89,7 @@ WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
 
 /*--------------------------------------------------------------------*/
 
-void
+static void
 WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace)
 {
 	struct worker *w, ww;
@@ -123,3 +130,384 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace)
 	HSH_Cleanup(w);
 	Pool_Sumstat(w);
 }
+
+/*--------------------------------------------------------------------
+ * Summing of stats into pool counters
+ */
+
+static void
+pool_addstat(struct dstat *dst, struct dstat *src)
+{
+
+	dst->summs++;
+#define L0(n)
+#define L1(n) (dst->n += src->n)
+#define VSC_F(n,t,l,s,f,v,d,e)	L##l(n);
+#include "tbl/vsc_f_main.h"
+#undef VSC_F
+#undef L0
+#undef L1
+	memset(src, 0, sizeof *src);
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct worker *
+pool_getidleworker(struct pool *pp)
+{
+	struct pool_task *pt;
+	struct worker *wrk;
+
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	Lck_AssertHeld(&pp->mtx);
+	pt = VTAILQ_FIRST(&pp->idle_queue);
+	if (pt == NULL) {
+		if (pp->nthr < cache_param->wthread_max) {
+			pp->dry++;
+			AZ(pthread_cond_signal(&pp->herder_cond));
+		}
+		return (NULL);
+	}
+	AZ(pt->func);
+	CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
+	return (wrk);
+}
+
+/*--------------------------------------------------------------------
+ * Special scheduling:  If no thread can be found, the current thread
+ * will be prepared for rescheduling instead.
+ * The selected threads workspace is reserved and the argument put there.
+ * Return one if another thread was scheduled, otherwise zero.
+ */
+
+int
+Pool_Task_Arg(struct worker *wrk, task_func_t *func,
+    const void *arg, size_t arg_len)
+{
+	struct pool *pp;
+	struct worker *wrk2;
+	int retval;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AN(arg);
+	AN(arg_len);
+	pp = wrk->pool;
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+
+	Lck_Lock(&pp->mtx);
+	wrk2 = pool_getidleworker(pp);
+	if (wrk2 != NULL) {
+		VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list);
+		retval = 1;
+	} else {
+		wrk2 = wrk;
+		retval = 0;
+	}
+	Lck_Unlock(&pp->mtx);
+	AZ(wrk2->task.func);
+
+	assert(arg_len <= WS_Reserve(wrk2->aws, arg_len));
+	memcpy(wrk2->aws->f, arg, arg_len);
+	wrk2->task.func = func;
+	wrk2->task.priv = wrk2->aws->f;
+	if (retval)
+		AZ(pthread_cond_signal(&wrk2->cond));
+	return (retval);
+}
+
+/*--------------------------------------------------------------------
+ * Enter a new task to be done
+ */
+
+int
+Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
+{
+	struct worker *wrk;
+	int retval = 0;
+
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	AN(task);
+	AN(task->func);
+
+	Lck_Lock(&pp->mtx);
+
+	/*
+	 * The common case first:  Take an idle thread, do it.
+	 */
+
+	wrk = pool_getidleworker(pp);
+	if (wrk != NULL) {
+		VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
+		AZ(wrk->task.func);
+		wrk->task.func = task->func;
+		wrk->task.priv = task->priv;
+		Lck_Unlock(&pp->mtx);
+		AZ(pthread_cond_signal(&wrk->cond));
+		return (0);
+	}
+
+	switch (how) {
+	case POOL_NO_QUEUE:
+		retval = -1;
+		break;
+	case POOL_QUEUE_FRONT:
+		/* If we have too much in the queue already, refuse. */
+		if (pp->lqueue > cache_param->wthread_queue_limit) {
+			pp->ndropped++;
+			retval = -1;
+		} else {
+			VTAILQ_INSERT_TAIL(&pp->front_queue, task, list);
+			pp->nqueued++;
+			pp->lqueue++;
+		}
+		break;
+	case POOL_QUEUE_BACK:
+		VTAILQ_INSERT_TAIL(&pp->back_queue, task, list);
+		break;
+	default:
+		WRONG("Unknown enum pool_how");
+	}
+	Lck_Unlock(&pp->mtx);
+	return (retval);
+}
+
+/*--------------------------------------------------------------------
+ * Empty function used as a pointer value for the thread exit condition.
+ */
+
+static void __match_proto__(task_func_t)
+pool_kiss_of_death(struct worker *wrk, void *priv)
+{
+	(void)wrk;
+	(void)priv;
+}
+
+
+/*--------------------------------------------------------------------
+ * This is the work function for worker threads in the pool.
+ */
+
+static void
+Pool_Work_Thread(struct pool *pp, struct worker *wrk)
+{
+	struct pool_task *tp;
+	struct pool_task tpx, tps;
+	int i;
+
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	wrk->pool = pp;
+	while (1) {
+		Lck_Lock(&pp->mtx);
+
+		CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+
+		WS_Reset(wrk->aws, NULL);
+		AZ(wrk->vsl);
+
+		tp = VTAILQ_FIRST(&pp->front_queue);
+		if (tp != NULL) {
+			pp->lqueue--;
+			VTAILQ_REMOVE(&pp->front_queue, tp, list);
+		} else {
+			tp = VTAILQ_FIRST(&pp->back_queue);
+			if (tp != NULL)
+				VTAILQ_REMOVE(&pp->back_queue, tp, list);
+		}
+
+		if ((tp == NULL && wrk->stats->summs > 0) ||
+		    (wrk->stats->summs >= cache_param->wthread_stats_rate))
+			pool_addstat(pp->a_stat, wrk->stats);
+
+		if (tp != NULL) {
+			wrk->stats->summs++;
+		} else if (pp->b_stat != NULL && pp->a_stat->summs) {
+			/* Nothing to do, push pool stats into global pool */
+			tps.func = pool_stat_summ;
+			tps.priv = pp->a_stat;
+			pp->a_stat = pp->b_stat;
+			pp->b_stat = NULL;
+			tp = &tps;
+		} else {
+			/* Nothing to do: To sleep, perchance to dream ... */
+			if (isnan(wrk->lastused))
+				wrk->lastused = VTIM_real();
+			wrk->task.func = NULL;
+			wrk->task.priv = wrk;
+			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
+			do {
+				i = Lck_CondWait(&wrk->cond, &pp->mtx,
+				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
+				if (i == ETIMEDOUT)
+					VCL_Rel(&wrk->vcl);
+			} while (wrk->task.func == NULL);
+			tpx = wrk->task;
+			tp = &tpx;
+			wrk->stats->summs++;
+		}
+		Lck_Unlock(&pp->mtx);
+
+		if (tp->func == pool_kiss_of_death)
+			break;
+
+		do {
+			memset(&wrk->task, 0, sizeof wrk->task);
+			assert(wrk->pool == pp);
+			tp->func(wrk, tp->priv);
+			tpx = wrk->task;
+			tp = &tpx;
+		} while (tp->func != NULL);
+
+		/* cleanup for next task */
+		wrk->seen_methods = 0;
+	}
+	wrk->pool = NULL;
+}
+
+/*--------------------------------------------------------------------
+ * Create another worker thread.
+ */
+
+struct pool_info {
+	unsigned		magic;
+#define POOL_INFO_MAGIC		0x4e4442d3
+	size_t			stacksize;
+	struct pool		*qp;
+};
+
+static void *
+pool_thread(void *priv)
+{
+	struct pool_info *pi;
+
+	CAST_OBJ_NOTNULL(pi, priv, POOL_INFO_MAGIC);
+	WRK_Thread(pi->qp, pi->stacksize, cache_param->workspace_thread);
+	FREE_OBJ(pi);
+	return (NULL);
+}
+
+static void
+pool_breed(struct pool *qp)
+{
+	pthread_t tp;
+	pthread_attr_t tp_attr;
+	struct pool_info *pi;
+
+	AZ(pthread_attr_init(&tp_attr));
+	AZ(pthread_attr_setdetachstate(&tp_attr, PTHREAD_CREATE_DETACHED));
+
+	/* Set the stacksize for worker threads we create */
+	if (cache_param->wthread_stacksize != UINT_MAX)
+		AZ(pthread_attr_setstacksize(&tp_attr,
+		    cache_param->wthread_stacksize));
+
+	ALLOC_OBJ(pi, POOL_INFO_MAGIC);
+	AN(pi);
+	AZ(pthread_attr_getstacksize(&tp_attr, &pi->stacksize));
+	pi->qp = qp;
+
+	if (pthread_create(&tp, &tp_attr, pool_thread, pi)) {
+		VSL(SLT_Debug, 0, "Create worker thread failed %d %s",
+		    errno, strerror(errno));
+		Lck_Lock(&pool_mtx);
+		VSC_C_main->threads_failed++;
+		Lck_Unlock(&pool_mtx);
+		VTIM_sleep(cache_param->wthread_fail_delay);
+	} else {
+		qp->dry = 0;
+		qp->nthr++;
+		Lck_Lock(&pool_mtx);
+		VSC_C_main->threads++;
+		VSC_C_main->threads_created++;
+		Lck_Unlock(&pool_mtx);
+		VTIM_sleep(cache_param->wthread_add_delay);
+	}
+
+	AZ(pthread_attr_destroy(&tp_attr));
+}
+
+/*--------------------------------------------------------------------
+ * Herd a single pool
+ *
+ * This thread wakes up whenever a pool queues.
+ *
+ * The trick here is to not be too aggressive about creating threads.
+ * We do this by only examining one pool at a time, and by sleeping
+ * a short while whenever we create a thread and a little while longer
+ * whenever we fail to, hopefully missing a lot of cond_signals in
+ * the meantime.
+ *
+ * XXX: probably need a lot more work.
+ *
+ */
+
+void*
+pool_herder(void *priv)
+{
+	struct pool *pp;
+	struct pool_task *pt;
+	double t_idle;
+	struct worker *wrk;
+
+	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
+
+	while (1) {
+		/* Make more threads if needed and allowed */
+		if (pp->nthr < cache_param->wthread_min ||
+		    (pp->dry && pp->nthr < cache_param->wthread_max)) {
+			pool_breed(pp);
+			continue;
+		}
+		assert(pp->nthr >= cache_param->wthread_min);
+
+		if (pp->nthr > cache_param->wthread_min) {
+
+			t_idle = VTIM_real() - cache_param->wthread_timeout;
+
+			Lck_Lock(&pp->mtx);
+			/* XXX: unsafe counters */
+			VSC_C_main->sess_queued += pp->nqueued;
+			VSC_C_main->sess_dropped += pp->ndropped;
+			pp->nqueued = pp->ndropped = 0;
+
+			wrk = NULL;
+			pt = VTAILQ_LAST(&pp->idle_queue, taskhead);
+			if (pt != NULL) {
+				AZ(pt->func);
+				CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
+
+				if (wrk->lastused < t_idle ||
+				    pp->nthr > cache_param->wthread_max) {
+					/* Give it a kiss on the cheek... */
+					VTAILQ_REMOVE(&pp->idle_queue,
+					    &wrk->task, list);
+					wrk->task.func = pool_kiss_of_death;
+					AZ(pthread_cond_signal(&wrk->cond));
+				} else
+					wrk = NULL;
+			}
+			Lck_Unlock(&pp->mtx);
+
+			if (wrk != NULL) {
+				pp->nthr--;
+				Lck_Lock(&pool_mtx);
+				VSC_C_main->threads--;
+				VSC_C_main->threads_destroyed++;
+				Lck_Unlock(&pool_mtx);
+				VTIM_sleep(cache_param->wthread_destroy_delay);
+				continue;
+			}
+		}
+
+		Lck_Lock(&pp->mtx);
+		if (!pp->dry) {
+			(void)Lck_CondWait(&pp->herder_cond, &pp->mtx,
+				VTIM_real() + 5);
+		} else {
+			/* XXX: unsafe counters */
+			VSC_C_main->threads_limited++;
+			pp->dry = 0;
+		}
+		Lck_Unlock(&pp->mtx);
+	}
+	NEEDLESS_RETURN(NULL);
+}



From phk at FreeBSD.org  Tue May 26 20:36:43 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 26 May 2015 22:36:43 +0200
Subject: [master] 841abfe Absorb struct sesspool into struct pool
Message-ID: <E1YxLad-00048F-Vm@project.varnish-software.com>


commit 841abfecb197a9182cf949e36d2debc31280a7ec
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 26 20:36:22 2015 +0000

    Absorb struct sesspool into struct pool

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bb3c61d..399fdcb 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -616,16 +616,6 @@ struct req {
  * works, is not realistic without a lot of code changes.
  */
 
-struct sesspool {
-	unsigned		magic;
-#define SESSPOOL_MAGIC		0xd916e202
-	struct pool		*pool;
-	struct mempool		*mpl_req;
-	struct mempool		*mpl_sess;
-
-	struct waiter		*http1_waiter;
-};
-
 enum sess_attr {
 #define SESS_ATTR(UP, low, typ, len)	SA_##UP,
 #include "tbl/sess_attr.h"
@@ -644,7 +634,7 @@ struct sess {
 
 	/* Cross references ------------------------------------------*/
 
-	struct sesspool		*sesspool;
+	struct pool		*pool;
 
 	struct waited		waited;
 
@@ -686,7 +676,6 @@ typedef enum htc_status_e htc_complete_f(struct http_conn *);
 /* cache_acceptor.c */
 void VCA_Init(void);
 void VCA_Shutdown(void);
-void VCA_New_SessPool(struct pool *pp, struct sesspool *sp);
 
 /* cache_backend_cfg.c */
 void VBE_InitCfg(void);
@@ -990,12 +979,11 @@ int Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req);
 void Req_Fail(struct req *req, enum sess_close reason);
 
 /* cache_session.c [SES] */
-struct sess *SES_New(struct sesspool *);
+struct sess *SES_New(struct pool *);
 void SES_Close(struct sess *sp, enum sess_close reason);
 void SES_Wait(struct sess *sp);
 void SES_Delete(struct sess *sp, enum sess_close reason, double now);
-struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no);
-void SES_DeletePool(struct sesspool *sp);
+void SES_NewPool(struct pool *pp, unsigned pool_no);
 int SES_Reschedule_Req(struct req *);
 task_func_t SES_Proto_Sess;
 task_func_t SES_Proto_Req;
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index f4ead24..45a1828 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -41,6 +41,7 @@
 #include <netinet/tcp.h>
 
 #include "cache.h"
+#include "cache_pool.h"
 #include "common/heritage.h"
 
 #include "vcli.h"
@@ -63,7 +64,6 @@ struct wrk_accept {
 	socklen_t		acceptaddrlen;
 	int			acceptsock;
 	struct listen_sock	*acceptlsock;
-	struct sesspool		*sesspool;
 };
 
 struct poolsock {
@@ -71,7 +71,7 @@ struct poolsock {
 #define POOLSOCK_MAGIC			0x1b0a2d38
 	struct listen_sock		*lsock;
 	struct pool_task		task;
-	struct sesspool			*sesspool;
+	struct pool			*pool;
 };
 
 /*--------------------------------------------------------------------
@@ -291,7 +291,6 @@ vca_pace_good(void)
 static void __match_proto__(task_func_t)
 vca_make_session(struct worker *wrk, void *arg)
 {
-	struct sesspool *pp;
 	struct sess *sp;
 	struct wrk_accept *wa;
 	struct sockaddr_storage ss;
@@ -304,11 +303,10 @@ vca_make_session(struct worker *wrk, void *arg)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(wa, arg, WRK_ACCEPT_MAGIC);
-	pp = wa->sesspool;
 
 	/* Turn accepted socket into a session */
 	AN(wrk->aws->r);
-	sp = SES_New(pp);
+	sp = SES_New(wrk->pool);
 	if (sp == NULL) {
 		/*
 		 * We consider this a DoS situation and silently close the
@@ -397,7 +395,6 @@ vca_accept_task(struct worker *wrk, void *arg)
 
 	while (1) {
 		INIT_OBJ(&wa, WRK_ACCEPT_MAGIC);
-		wa.sesspool = ps->sesspool;
 		wa.acceptlsock = ls;
 
 		vca_pace_check();
@@ -459,7 +456,7 @@ vca_accept_task(struct worker *wrk, void *arg)
  */
 
 void
-VCA_New_SessPool(struct pool *pp, struct sesspool *sp)
+VCA_NewPool(struct pool *pp)
 {
 	struct listen_sock *ls;
 	struct poolsock *ps;
@@ -470,7 +467,7 @@ VCA_New_SessPool(struct pool *pp, struct sesspool *sp)
 		ps->lsock = ls;
 		ps->task.func = vca_accept_task;
 		ps->task.priv = ps;
-		ps->sesspool = sp;
+		ps->pool = pp;
 		AZ(Pool_Task(pp, &ps->task, POOL_QUEUE_BACK));
 	}
 }
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index f71980d..628b18c 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -145,8 +145,8 @@ pool_mkpool(unsigned pool_no)
 	while (VTAILQ_EMPTY(&pp->idle_queue))
 		(void)usleep(10000);
 
-	pp->sesspool = SES_NewPool(pp, pool_no);
-	AN(pp->sesspool);
+	SES_NewPool(pp, pool_no);
+	VCA_NewPool(pp);
 
 	return (pp);
 }
@@ -182,8 +182,13 @@ pool_poolherder(void *priv)
 			}
 		}
 		/* XXX: remove pools */
-		if (0)
-			SES_DeletePool(NULL);
+		if (0) {
+			pp = VTAILQ_FIRST(&pools);
+			AN(pp);
+			MPL_Destroy(&pp->mpl_sess);
+			MPL_Destroy(&pp->mpl_req);
+			INCOMPL();
+		}
 		(void)sleep(1);
 		u = 0;
 		VTAILQ_FOREACH(pp, &pools, list)
diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
index 76acc08..ea343e6 100644
--- a/bin/varnishd/cache/cache_pool.h
+++ b/bin/varnishd/cache/cache_pool.h
@@ -50,11 +50,16 @@ struct pool {
 	unsigned			lqueue;
 	uintmax_t			ndropped;
 	uintmax_t			nqueued;
-	struct sesspool			*sesspool;
 	struct dstat			*a_stat;
 	struct dstat			*b_stat;
+
+	struct waitfor			wf;
+	struct mempool			*mpl_req;
+	struct mempool			*mpl_sess;
+	struct waiter			*waiter;
 };
 
 void *pool_herder(void*);
 task_func_t pool_stat_summ;
 extern struct lock			pool_mtx;
+void VCA_NewPool(struct pool *pp);
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 03c3557..4f02e26 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -38,6 +38,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "cache_pool.h"
 
 #include "vtim.h"
 
@@ -48,7 +49,7 @@
 struct req *
 Req_New(const struct worker *wrk, struct sess *sp)
 {
-	struct sesspool *pp;
+	struct pool *pp;
 	struct req *req;
 	uint16_t nhttp;
 	unsigned sz, hl;
@@ -56,9 +57,8 @@ Req_New(const struct worker *wrk, struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	pp = sp->sesspool;
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-	AN(pp->pool);
+	pp = sp->pool;
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 
 	req = MPL_Get(pp->mpl_req, &sz);
 	AN(req);
@@ -114,7 +114,7 @@ void
 Req_Release(struct req *req)
 {
 	struct sess *sp;
-	struct sesspool *pp;
+	struct pool *pp;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
@@ -129,9 +129,8 @@ Req_Release(struct req *req)
 		VSL_End(req->vsl);
 	sp = req->sp;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	pp = sp->sesspool;
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-	AN(pp->pool);
+	pp = sp->pool;
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	MPL_AssertSane(req);
 	VSL_Flush(req->vsl, 0);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index d49decc..76d47b8 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -46,6 +46,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "cache_pool.h"
 
 #include "vsa.h"
 #include "vtcp.h"
@@ -299,16 +300,16 @@ SES_RxReq(const struct worker *wrk, struct req *req, htc_complete_f *func)
  */
 
 struct sess *
-SES_New(struct sesspool *pp)
+SES_New(struct pool *pp)
 {
 	struct sess *sp;
 	unsigned sz;
 	char *p, *e;
 
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	sp = MPL_Get(pp->mpl_sess, &sz);
 	sp->magic = SESS_MAGIC;
-	sp->sesspool = pp;
+	sp->pool = pp;
 	memset(sp->sattr, 0xff, sizeof sp->sattr);
 
 	e = (char*)sp + sz;
@@ -403,19 +404,18 @@ int
 SES_Reschedule_Req(struct req *req)
 {
 	struct sess *sp;
-	struct sesspool *pp;
+	struct pool *pp;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	sp = req->sp;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	pp = sp->sesspool;
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-	AN(pp->pool);
+	pp = sp->pool;
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 
 	req->task.func = SES_Proto_Req;
 	req->task.priv = req;
 
-	return (Pool_Task(pp->pool, &req->task, POOL_QUEUE_FRONT));
+	return (Pool_Task(pp, &req->task, POOL_QUEUE_FRONT));
 }
 
 /*--------------------------------------------------------------------
@@ -426,7 +426,7 @@ static void __match_proto__(waiter_handle_f)
 ses_handle(struct waited *wp, enum wait_event ev, double now)
 {
 	struct sess *sp;
-	struct sesspool *pp;
+	struct pool *pp;
 	struct pool_task *tp;
 
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
@@ -442,14 +442,13 @@ ses_handle(struct waited *wp, enum wait_event ev, double now)
 		SES_Delete(sp, SC_REM_CLOSE, now);
 		break;
 	case WAITER_ACTION:
-		pp = sp->sesspool;
-		CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-		AN(pp->pool);
+		pp = sp->pool;
+		CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 		assert(sizeof *tp == WS_Reserve(sp->ws, sizeof *tp));
 		tp = (void*)sp->ws->f;
 		tp->func = SES_Proto_Sess;
 		tp->priv = sp;
-		if (Pool_Task(pp->pool, tp, POOL_QUEUE_FRONT))
+		if (Pool_Task(pp, tp, POOL_QUEUE_FRONT))
 			SES_Delete(sp, SC_OVERLOAD, now);
 		break;
 	case WAITER_CLOSE:
@@ -466,11 +465,11 @@ ses_handle(struct waited *wp, enum wait_event ev, double now)
 void
 SES_Wait(struct sess *sp)
 {
-	struct sesspool *pp;
+	struct pool *pp;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	pp = sp->sesspool;
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+	pp = sp->pool;
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	/*
 	 * XXX: waiter_epoll prevents us from zeroing the struct because
 	 * XXX: it keeps state across calls.
@@ -483,7 +482,7 @@ SES_Wait(struct sess *sp)
 	sp->waited.fd = sp->fd;
 	sp->waited.ptr = sp;
 	sp->waited.idle = sp->t_idle;
-	if (Wait_Enter(pp->http1_waiter, &sp->waited))
+	if (Wait_Enter(pp->waiter, &sp->waited))
 		SES_Delete(sp, SC_PIPE_OVERFLOW, NAN);
 }
 
@@ -541,12 +540,11 @@ SES_Close(struct sess *sp, enum sess_close reason)
 void
 SES_Delete(struct sess *sp, enum sess_close reason, double now)
 {
-	struct sesspool *pp;
+	struct pool *pp;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	pp = sp->sesspool;
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-	AN(pp->pool);
+	pp = sp->pool;
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 
 	if (reason != SC_NULL)
 		SES_Close(sp, reason);
@@ -573,17 +571,12 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
  * Create and delete pools
  */
 
-static struct waitfor ses_wf;
-
-struct sesspool *
-SES_NewPool(struct pool *wp, unsigned pool_no)
+void
+SES_NewPool(struct pool *pp, unsigned pool_no)
 {
-	struct sesspool *pp;
 	char nb[8];
 
-	ALLOC_OBJ(pp, SESSPOOL_MAGIC);
-	AN(pp);
-	pp->pool = wp;
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	bprintf(nb, "req%u", pool_no);
 	pp->mpl_req = MPL_New(nb, &cache_param->req_pool,
 	    &cache_param->workspace_client);
@@ -591,23 +584,8 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool,
 	    &cache_param->workspace_session);
 
-	INIT_OBJ(&ses_wf, WAITFOR_MAGIC);
-	ses_wf.func = ses_handle;
-	ses_wf.tmo = &cache_param->timeout_idle;
-	pp->http1_waiter = Waiter_New(&ses_wf);
-
-	VCA_New_SessPool(wp, pp);
-	return (pp);
-}
-
-void
-SES_DeletePool(struct sesspool *pp)
-{
-
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-	MPL_Destroy(&pp->mpl_sess);
-	MPL_Destroy(&pp->mpl_req);
-	/* Delete session pool must stop acceptor threads */
-	FREE_OBJ(pp);
-	INCOMPL();
+	INIT_OBJ(&pp->wf, WAITFOR_MAGIC);
+	pp->wf.func = ses_handle;
+	pp->wf.tmo = &cache_param->timeout_idle;
+	pp->waiter = Waiter_New(&pp->wf);
 }



From dridi at varni.sh  Tue May 26 22:56:28 2015
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Wed, 27 May 2015 00:56:28 +0200
Subject: [master] 49712a5 vmods running vmodtool require python, so check
 for it
In-Reply-To: <CAJV_h0YL91CVT2pKkdkxDyMpn8xx6MAwMicR1+SkPnoPXYp2=Q@mail.gmail.com>
References: <E1YvUBv-0005Lu-02@project.varnish-software.com>
 <CAJV_h0Y0Eomez5AATkpik+L3kmYQ-AD3cJ9kLgYKsJCXiCP6Fg@mail.gmail.com>
 <555E2417.4020104@uplex.de>
 <CAJV_h0bLfPhM7z7dQzudxPDH8=BergobS9210AQWPOgecXdTgg@mail.gmail.com>
 <555E5690.8090706@schokola.de>
 <CAJV_h0bcVmSnZEaRXymc05jbD+9WFKkG0Es61Yym4HTCp+r+fQ@mail.gmail.com>
 <555F2F8D.9030402@uplex.de>
 <CAJV_h0bvYpJ3DuQ+-z-jP_f+wkkOQyLY9pjdmP4bs+WyjSA9rw@mail.gmail.com>
 <5561A55B.4040608@uplex.de> <35990.1432465498@critter.freebsd.dk>
 <CA+nLBF0hNwLoTbEpQ=aP27KbiVbV6f-Uuc3Ffh-_e-2Q5_-1yg@mail.gmail.com>
 <CABoVN9AO-f6n0X=_TEaZQMZi61JPVhSDB47XiHthVpuMiavEmQ@mail.gmail.com>
 <CAJV_h0YL91CVT2pKkdkxDyMpn8xx6MAwMicR1+SkPnoPXYp2=Q@mail.gmail.com>
Message-ID: <CABoVN9B4+JznUDaKHpupVPWVw7fD3eN8JLEDg9eTZ32QwQ5E1g@mail.gmail.com>

On Tue, May 26, 2015 at 10:31 PM, Federico Schwindt <fgsch at lodoss.net> wrote:
> I'd say that's a bug.

Agreed, I have file an issue for Fedora:
https://bugzilla.redhat.com/show_bug.cgi?id=1225243



From phk at FreeBSD.org  Wed May 27 21:38:32 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 27 May 2015 23:38:32 +0200
Subject: [master] 15d4d99 remove two unused fields
Message-ID: <E1Yxj20-0002yT-DM@project.varnish-software.com>


commit 15d4d99fc153dcca540c2e54fdb525e7eba32eca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 27 06:51:55 2015 +0000

    remove two unused fields

diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h
index 5890912..b4d9078 100644
--- a/bin/varnishd/waiter/waiter_priv.h
+++ b/bin/varnishd/waiter/waiter_priv.h
@@ -38,12 +38,8 @@ struct waiter {
 	VTAILQ_ENTRY(waiter)		list;
 	VTAILQ_HEAD(,waited)		waithead;
 
-	int				dismantle;
-
 	struct waitfor			*waitfor;
 
-	double				next_idle;
-
 	void				*priv;
 };
 



From phk at FreeBSD.org  Wed May 27 21:38:32 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 27 May 2015 23:38:32 +0200
Subject: [master] f2d75dc Constify the comparison function prototype
Message-ID: <E1Yxj20-0002yZ-Eh@project.varnish-software.com>


commit f2d75dc30238d7b2677b20f3ce60f7ff87f6837f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 27 07:23:51 2015 +0000

    Constify the comparison function prototype

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 4db9537..e74b328 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -550,10 +550,10 @@ exp_expire(struct exp_priv *ep, double now)
  * object expires, accounting also for graceability, it is killed.
  */
 
-static int
-object_cmp(void *priv, void *a, void *b)
+static int __match_proto__(binheap_cmp_t)
+object_cmp(void *priv, const void *a, const void *b)
 {
-	struct objcore *aa, *bb;
+	const struct objcore *aa, *bb;
 
 	(void)priv;
 	CAST_OBJ_NOTNULL(aa, a, OBJCORE_MAGIC);
@@ -561,7 +561,7 @@ object_cmp(void *priv, void *a, void *b)
 	return (aa->timer_when < bb->timer_when);
 }
 
-static void
+static void __match_proto__(binheap_update_t)
 object_update(void *priv, void *p, unsigned u)
 {
 	struct objcore *oc;
diff --git a/include/binary_heap.h b/include/binary_heap.h
index d987104..7cef493 100644
--- a/include/binary_heap.h
+++ b/include/binary_heap.h
@@ -35,7 +35,7 @@
 
 struct binheap;
 
-typedef int binheap_cmp_t(void *priv, void *a, void *b);
+typedef int binheap_cmp_t(void *priv, const void *a, const void *b);
 	/*
 	 * Comparison function.
 	 * Should return true if item 'a' should be closer to the root
diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c
index 6429e4b..0ceaaa9 100644
--- a/lib/libvarnish/vev.c
+++ b/lib/libvarnish/vev.c
@@ -38,6 +38,7 @@
 #include <string.h>
 #include <time.h>
 
+#include "vdef.h"
 #include "miniobj.h"
 #include "vas.h"
 
@@ -93,7 +94,7 @@ struct vev_base {
 
 /*--------------------------------------------------------------------*/
 
-static void
+static void __match_proto__(binheap_update_t)
 vev_bh_update(void *priv, void *a, unsigned u)
 {
 	struct vev_base *evb;
@@ -104,11 +105,11 @@ vev_bh_update(void *priv, void *a, unsigned u)
 	e->__binheap_idx = u;
 }
 
-static int
-vev_bh_cmp(void *priv, void *a, void *b)
+static int __match_proto__(binheap_cmp_t)
+vev_bh_cmp(void *priv, const void *a, const void *b)
 {
 	struct vev_base *evb;
-	struct vev *ea, *eb;
+	const struct vev *ea, *eb;
 
 	CAST_OBJ_NOTNULL(evb, priv, VEV_BASE_MAGIC);
 	CAST_OBJ_NOTNULL(ea, a, VEV_MAGIC);



From phk at FreeBSD.org  Wed May 27 21:38:32 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 27 May 2015 23:38:32 +0200
Subject: [master] 0f03834 Give the kqueue waiter a binheap to track timeouts.
Message-ID: <E1Yxj20-0002yi-Fe@project.varnish-software.com>


commit 0f038349f7532249c56ccf12850b5ec0bd9dcd29
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 27 09:08:19 2015 +0000

    Give the kqueue waiter a binheap to track timeouts.

diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index 46957be..d3a33ca 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -34,11 +34,80 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+#include "binary_heap.h"
+
 #include "cache/cache.h"
 
 #include "waiter/waiter_priv.h"
 #include "waiter/mgt_waiter.h"
 
+static int __match_proto__(binheap_cmp_t)
+waited_cmp(void *priv, const void *a, const void *b)
+{
+	const struct waiter *ww;
+	const struct waited *aa, *bb;
+
+	CAST_OBJ_NOTNULL(ww, priv, WAITER_MAGIC);
+	CAST_OBJ_NOTNULL(aa, a, WAITED_MAGIC);
+	CAST_OBJ_NOTNULL(bb, b, WAITED_MAGIC);
+
+	return (aa->idle + Wait_Tmo(ww, aa) < bb->idle + Wait_Tmo(ww, bb));
+}
+
+static void __match_proto__(binheap_update_t)
+waited_update(void *priv, void *p, unsigned u)
+{
+	struct waited *pp;
+
+	(void)priv;
+	CAST_OBJ_NOTNULL(pp, p, WAITED_MAGIC);
+	pp->idx = u;
+}
+
+/**********************************************************************/
+
+void
+Wait_Call(const struct waiter *w,
+    struct waited *wp, enum wait_event ev, double now)
+{
+	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
+	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
+	CHECK_OBJ_NOTNULL(w->waitfor, WAITFOR_MAGIC);
+	AN(w->waitfor->func);
+	if (wp->idx != BINHEAP_NOIDX)
+		binheap_delete(w->heap, wp->idx);
+	assert(wp->idx == BINHEAP_NOIDX);
+	w->waitfor->func(wp, ev, now);
+}
+
+/**********************************************************************/
+
+void
+Wait_HeapInsert(const struct waiter *w, struct waited *wp)
+{
+	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
+	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
+	assert(wp->idx == BINHEAP_NOIDX);
+	binheap_insert(w->heap, wp);
+}
+
+double
+Wait_HeapDue(const struct waiter *w, struct waited **wpp)
+{
+	struct waited *wp;
+
+	wp = binheap_root(w->heap);
+	CHECK_OBJ_ORNULL(wp, WAITED_MAGIC);
+	if (wp == NULL) {
+		*wpp = NULL;
+		return (0);
+	}
+	*wpp = wp;
+	return(wp->idle + Wait_Tmo(w, wp));
+}
+
+/**********************************************************************/
+
 int
 Wait_Enter(const struct waiter *w, struct waited *wp)
 {
@@ -46,6 +115,7 @@ Wait_Enter(const struct waiter *w, struct waited *wp)
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 	assert(wp->fd > 0);			// stdin never comes here
+	wp->idx = BINHEAP_NOIDX;
 	return (w->impl->enter(w->priv, wp));
 }
 
@@ -79,6 +149,7 @@ Waiter_New(struct waitfor *wf)
 	w->impl = waiter;
 	w->waitfor = wf;
 	VTAILQ_INIT(&w->waithead);
+	w->heap = binheap_new(w, waited_cmp, waited_update);
 
 	waiter->init(w);
 
@@ -95,6 +166,7 @@ Waiter_Destroy(struct waiter **wp)
 	*wp = NULL;
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 
+	AZ(binheap_root(w->heap));
 	AN(w->impl->fini);
 	w->impl->fini(w);
 	FREE_OBJ(w);
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index a6d4c8c..f8af96e 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -52,6 +52,8 @@ struct vwk {
 	int			kq;
 	struct waiter		*waiter;
 	pthread_t		thread;
+	double			next;
+	int			pipe[2];
 
 	VTAILQ_HEAD(,waited)	list;
 	struct lock		mtx;
@@ -66,58 +68,67 @@ vwk_thread(void *priv)
 	struct vwk *vwk;
 	struct kevent ke[NKEV], *kp;
 	int j, n;
-	double now, idle, last_idle;
+	double now, then;
 	struct timespec ts;
-	struct waited *wp, *wp2;
+	struct waited *wp;
+	struct waiter *w;
+	char c;
 
 	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
+	w = vwk->waiter;
+	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	THR_SetName("cache-kqueue");
 
-	last_idle = 0.0;
+	now = VTIM_real();
+	Lck_Lock(&vwk->mtx);
 	while (1) {
-		now = .3 * Wait_Tmo(vwk->waiter, NULL);
-		ts.tv_sec = (time_t)floor(now);
-		ts.tv_nsec = (long)(1e9 * (now - ts.tv_sec));
+		while (1) {
+			/*
+			 * XXX: We could avoid many syscalls here if we were
+			 * XXX: allowed to just close the fd's on timeout.
+			 */
+			then = Wait_HeapDue(w, &wp);
+			if (wp == NULL) {
+				vwk->next = now + 100;
+				break;
+			} else if (then > now) {
+				vwk->next = then;
+				break;
+			}
+			CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
+			EV_SET(ke, wp->fd, EVFILT_READ, EV_DELETE, 0, 0, NULL);
+			AZ(kevent(vwk->kq, ke, 1, NULL, 0, NULL));
+			Wait_Call(w, wp, WAITER_TIMEOUT, now);
+		}
+		then = vwk->next - now;
+		ts.tv_sec = (time_t)floor(then);
+		ts.tv_nsec = (long)(1e9 * (then - ts.tv_sec));
+		Lck_Unlock(&vwk->mtx);
 		n = kevent(vwk->kq, NULL, 0, ke, NKEV, &ts);
-		if (n < 0 && vwk->die)
-			break;
 		assert(n >= 0);
 		assert(n <= NKEV);
 		now = VTIM_real();
+		Lck_Lock(&vwk->mtx);
 		for (kp = ke, j = 0; j < n; j++, kp++) {
 			assert(kp->filter == EVFILT_READ);
+			if (ke[j].udata == vwk) {
+				assert(read(vwk->pipe[0], &c, 1) == 1);
+				continue;
+			}
 			CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC);
-			Lck_Lock(&vwk->mtx);
 			VTAILQ_REMOVE(&vwk->list, wp, list);
-			Lck_Unlock(&vwk->mtx);
 			if (kp->flags & EV_EOF)
-				Wait_Call(vwk->waiter, wp, WAITER_REMCLOSE, now);
+				Wait_Call(w, wp, WAITER_REMCLOSE, now);
 			else
-				Wait_Call(vwk->waiter, wp, WAITER_ACTION, now);
-		}
-		idle = now - Wait_Tmo(vwk->waiter, NULL);
-		if (now - last_idle < .3 * Wait_Tmo(vwk->waiter, NULL))
-			continue;
-		last_idle = now;
-		n = 0;
-		Lck_Lock(&vwk->mtx);
-		VTAILQ_FOREACH_SAFE(wp, &vwk->list, list, wp2) {
-			if (wp->idle > idle)
-				continue;
-			EV_SET(ke + n, wp->fd,
-			    EVFILT_READ, EV_DELETE, 0, 0, wp);
-			if (++n == NKEV)
-				break;
+				Wait_Call(w, wp, WAITER_ACTION, now);
 		}
-		if (n > 0)
-			AZ(kevent(vwk->kq, ke, n, NULL, 0, NULL));
-		for (j = 0; j < n; j++) {
-			CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC);
-			VTAILQ_REMOVE(&vwk->list, wp, list);
-			Wait_Call(vwk->waiter, wp, WAITER_TIMEOUT, now);
-		}
-		Lck_Unlock(&vwk->mtx);
+		if (VTAILQ_EMPTY(&vwk->list) && vwk->die)
+			break;
 	}
+	Lck_Unlock(&vwk->mtx);
+	AZ(close(vwk->pipe[0]));
+	AZ(close(vwk->pipe[1]));
+	AZ(close(vwk->kq));
 	return(NULL);
 }
 
@@ -133,7 +144,13 @@ vwk_enter(void *priv, struct waited *wp)
 	EV_SET(&ke, wp->fd, EVFILT_READ, EV_ADD|EV_ONESHOT, 0, 0, wp);
 	Lck_Lock(&vwk->mtx);
 	VTAILQ_INSERT_TAIL(&vwk->list, wp, list);
+	Wait_HeapInsert(vwk->waiter, wp);
 	AZ(kevent(vwk->kq, &ke, 1, NULL, 0, NULL));
+
+	/* If the kqueue isn't due before our timeout, poke it via the pipe */
+	if (Wait_Tmo(vwk->waiter, wp) < vwk->next)
+		assert(write(vwk->pipe[1], "X", 1) == 1);
+
 	Lck_Unlock(&vwk->mtx);
 	return(0);
 }
@@ -144,6 +161,7 @@ static void __match_proto__(waiter_init_f)
 vwk_init(struct waiter *w)
 {
 	struct vwk *vwk;
+	struct kevent ke;
 
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	vwk = w->priv;
@@ -154,6 +172,9 @@ vwk_init(struct waiter *w)
 	assert(vwk->kq >= 0);
 	VTAILQ_INIT(&vwk->list);
 	Lck_New(&vwk->mtx, lck_misc);
+	AZ(pipe(vwk->pipe));
+	EV_SET(&ke, vwk->pipe[0], EVFILT_READ, EV_ADD, 0, 0, vwk);
+	AZ(kevent(vwk->kq, &ke, 1, NULL, 0, NULL));
 
 	AZ(pthread_create(&vwk->thread, NULL, vwk_thread, vwk));
 }
@@ -168,19 +189,11 @@ vwk_fini(struct waiter *w)
 {
 	struct vwk *vwk;
 	void *vp;
-	int i;
 
 	CAST_OBJ_NOTNULL(vwk, w->priv, VWK_MAGIC);
 	Lck_Lock(&vwk->mtx);
-	while (!VTAILQ_EMPTY(&vwk->list)) {
-		Lck_Unlock(&vwk->mtx);
-		(void)usleep(100000);
-		Lck_Lock(&vwk->mtx);
-	}
 	vwk->die = 1;
-	i = vwk->kq;
-	vwk->kq = -1;
-	AZ(close(i));
+	assert(write(vwk->pipe[1], "Y", 1) == 1);
 	Lck_Unlock(&vwk->mtx);
 	AZ(pthread_join(vwk->thread, &vp));
 	Lck_Delete(&vwk->mtx);
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
index 0e70d9e..7227fcd 100644
--- a/bin/varnishd/waiter/waiter.h
+++ b/bin/varnishd/waiter/waiter.h
@@ -65,6 +65,7 @@ struct waited {
 	unsigned		magic;
 #define WAITED_MAGIC		0x1743992d
 	int			fd;
+	unsigned		idx;
 	void			*ptr;
 	double			idle;
 	VTAILQ_ENTRY(waited)	list;
diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h
index b4d9078..5215995 100644
--- a/bin/varnishd/waiter/waiter_priv.h
+++ b/bin/varnishd/waiter/waiter_priv.h
@@ -30,6 +30,7 @@
  */
 
 struct waited;
+struct binheap;
 
 struct waiter {
 	unsigned			magic;
@@ -41,6 +42,7 @@ struct waiter {
 	struct waitfor			*waitfor;
 
 	void				*priv;
+	struct binheap			*heap;
 };
 
 typedef void waiter_init_f(struct waiter *);
@@ -68,13 +70,7 @@ Wait_Tmo(const struct waiter *w, const struct waited *wp)
 	return (*w->waitfor->tmo);
 }
 
-static inline void
-Wait_Call(const struct waiter *w,
-    struct waited *wp, enum wait_event ev, double now)
-{
-	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
-	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
-	CHECK_OBJ_NOTNULL(w->waitfor, WAITFOR_MAGIC);
-	AN(w->waitfor->func);
-	w->waitfor->func(wp, ev, now);
-}
+void Wait_Call(const struct waiter *, struct waited *,
+    enum wait_event ev, double now);
+void Wait_HeapInsert(const struct waiter *, struct waited *);
+double Wait_HeapDue(const struct waiter *, struct waited **);



From phk at FreeBSD.org  Wed May 27 21:38:32 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 27 May 2015 23:38:32 +0200
Subject: [master] 754f386 Make the poll waiter also use the binheap
Message-ID: <E1Yxj20-0002ys-Gp@project.varnish-software.com>


commit 754f386d663b66289cf75b664bad5671a50578dd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 27 10:54:00 2015 +0000

    Make the poll waiter also use the binheap

diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index d3a33ca..4d2f794 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -51,7 +51,7 @@ waited_cmp(void *priv, const void *a, const void *b)
 	CAST_OBJ_NOTNULL(aa, a, WAITED_MAGIC);
 	CAST_OBJ_NOTNULL(bb, b, WAITED_MAGIC);
 
-	return (aa->idle + Wait_Tmo(ww, aa) < bb->idle + Wait_Tmo(ww, bb));
+	return (Wait_When(ww, aa) < Wait_When(ww, bb));
 }
 
 static void __match_proto__(binheap_update_t)
@@ -99,11 +99,13 @@ Wait_HeapDue(const struct waiter *w, struct waited **wpp)
 	wp = binheap_root(w->heap);
 	CHECK_OBJ_ORNULL(wp, WAITED_MAGIC);
 	if (wp == NULL) {
-		*wpp = NULL;
+		if (wpp != NULL)
+			*wpp = NULL;
 		return (0);
 	}
-	*wpp = wp;
-	return(wp->idle + Wait_Tmo(w, wp));
+	if (wpp != NULL)
+		*wpp = wp;
+	return(Wait_When(w, wp));
 }
 
 /**********************************************************************/
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index f8af96e..ea5ea22 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -148,7 +148,7 @@ vwk_enter(void *priv, struct waited *wp)
 	AZ(kevent(vwk->kq, &ke, 1, NULL, 0, NULL));
 
 	/* If the kqueue isn't due before our timeout, poke it via the pipe */
-	if (Wait_Tmo(vwk->waiter, wp) < vwk->next)
+	if (Wait_When(vwk->waiter, wp) < vwk->next)
 		assert(write(vwk->pipe[1], "X", 1) == 1);
 
 	Lck_Unlock(&vwk->mtx);
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index 2de4402..5c56f9b 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -92,20 +92,22 @@ vwp_extend_pollspace(struct vwp *vwp)
 /*--------------------------------------------------------------------*/
 
 static void
-vwp_add(struct vwp *vwp, struct waited *w)
+vwp_add(struct vwp *vwp, struct waited *wp)
 {
 
-	CHECK_OBJ_NOTNULL(w, WAITED_MAGIC);
+VSL(SLT_Debug, wp->fd, "ADD");
+	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 	CHECK_OBJ_NOTNULL(vwp, VWP_MAGIC);
 	if (vwp->hpoll == vwp->npoll)
 		vwp_extend_pollspace(vwp);
 	assert(vwp->hpoll < vwp->npoll);
 	assert(vwp->pollfd[vwp->hpoll].fd == -1);
 	AZ(vwp->idx[vwp->hpoll]);
-	vwp->pollfd[vwp->hpoll].fd = w->fd;
+	vwp->pollfd[vwp->hpoll].fd = wp->fd;
 	vwp->pollfd[vwp->hpoll].events = POLLIN;
-	vwp->idx[vwp->hpoll] = w;
+	vwp->idx[vwp->hpoll] = wp;
 	vwp->hpoll++;
+	Wait_HeapInsert(vwp->waiter, wp);
 }
 
 static void
@@ -116,6 +118,7 @@ vwp_del(struct vwp *vwp, int n)
 		vwp->pollfd[n] = vwp->pollfd[vwp->hpoll];
 		vwp->idx[n] = vwp->idx[vwp->hpoll];
 	}
+VSL(SLT_Debug, vwp->pollfd[vwp->hpoll].fd, "DEL");
 	memset(&vwp->pollfd[vwp->hpoll], 0, sizeof(*vwp->pollfd));
 	vwp->pollfd[vwp->hpoll].fd = -1;
 	vwp->idx[vwp->hpoll] = NULL;
@@ -154,34 +157,34 @@ vwp_main(void *priv)
 	int v;
 	struct vwp *vwp;
 	struct waited *wp;
-	double now, idle;
-	int i, dopipe;
+	double now, then;
+	int i;
 
 	THR_SetName("cache-poll");
 	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
 
 	while (1) {
-		v = poll(vwp->pollfd, vwp->hpoll,
-		    (int)floor(1e3 * Wait_Tmo(vwp->waiter, NULL)));
+		then = Wait_HeapDue(vwp->waiter, &wp);
+		if (wp == NULL)
+			i = -1;
+		else
+			i = (int)floor(1e3 * (then - VTIM_real()));
+		v = poll(vwp->pollfd, vwp->hpoll, i);
 		assert(v >= 0);
-		if (v == 0)
-			v = vwp->hpoll;
 		now = VTIM_real();
-		idle = now - Wait_Tmo(vwp->waiter, NULL);
-		i = 0;
-		dopipe = 0;
-		while (v > 0 && i < vwp->hpoll) {
-			if (vwp->pollfd[i].revents)
-				v--;
-			if (vwp->pollfd[i].fd == vwp->pipes[0]) {
-				if (vwp->pollfd[i].revents)
-					dopipe = 1;
-				i++;
-				continue;
-			}
+		if (vwp->pollfd[0].revents)
+			v--;
+		for (i = 1; i < vwp->hpoll;) {
+			assert(vwp->pollfd[i].fd != vwp->pipes[0]);
 			wp = vwp->idx[i];
 			CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
-			if (wp->idle <= idle) {
+
+			if (v == 0 && Wait_HeapDue(vwp->waiter, NULL) > now)
+				break;
+			if (vwp->pollfd[i].revents)
+				v--;
+			then = Wait_When(vwp->waiter, wp);
+			if (then <= now) {
 				Wait_Call(vwp->waiter, wp, WAITER_TIMEOUT, now);
 				vwp_del(vwp, i);
 			} else if (vwp->pollfd[i].revents & POLLIN) {
@@ -193,7 +196,7 @@ vwp_main(void *priv)
 				i++;
 			}
 		}
-		if (dopipe)
+		if (vwp->pollfd[0].revents)
 			vwp_dopipe(vwp);
 	}
 	NEEDLESS_RETURN(NULL);
diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h
index 5215995..331e3ad 100644
--- a/bin/varnishd/waiter/waiter_priv.h
+++ b/bin/varnishd/waiter/waiter_priv.h
@@ -70,6 +70,12 @@ Wait_Tmo(const struct waiter *w, const struct waited *wp)
 	return (*w->waitfor->tmo);
 }
 
+static inline double
+Wait_When(const struct waiter *w, const struct waited *wp)
+{
+	return (Wait_Tmo(w, wp) + wp->idle);
+}
+
 void Wait_Call(const struct waiter *, struct waited *,
     enum wait_event ev, double now);
 void Wait_HeapInsert(const struct waiter *, struct waited *);



From phk at FreeBSD.org  Wed May 27 21:38:32 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 27 May 2015 23:38:32 +0200
Subject: [master] 1225301 Ditch the per tcp-pool waiters and recycle backend
 connections on the pool waiter for the working thread doing the recycling.
Message-ID: <E1Yxj20-0002z4-I3@project.varnish-software.com>


commit 1225301cfe89a682b096b820819256566e9b1c77
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 27 21:36:56 2015 +0000

    Ditch the per tcp-pool waiters and recycle backend connections on
    the pool waiter for the working thread doing the recycling.
    
    This does not restrict which threads can reuse the connection later on.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index d451943..e84ff06 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -184,7 +184,7 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 		    bp->display_name);
 		Lck_Lock(&bp->mtx);
 		VSC_C_main->backend_recycle++;
-		VBT_Recycle(bp->tcp_pool, &bo->htc->vbc);
+		VBT_Recycle(wrk, bp->tcp_pool, &bo->htc->vbc);
 	}
 #define ACCT(foo)	bp->vsc->foo += bo->acct.foo;
 #include "tbl/acct_fields_bereq.h"
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 7fdc55b..11f14bd 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -119,7 +119,7 @@ struct tcp_pool *VBT_Ref(const struct suckaddr *ip4,
     const struct suckaddr *ip6);
 void VBT_Rel(struct tcp_pool **tpp);
 int VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa);
-void VBT_Recycle(struct tcp_pool *tp, struct vbc **vbc);
+void VBT_Recycle(const struct worker *, struct tcp_pool *, struct vbc **);
 void VBT_Close(struct tcp_pool *tp, struct vbc **vbc);
 struct vbc *VBT_Get(struct tcp_pool *, double tmo, struct backend *,
     struct worker *);
diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c
index f130abd..70afa2e 100644
--- a/bin/varnishd/cache/cache_backend_tcp.c
+++ b/bin/varnishd/cache/cache_backend_tcp.c
@@ -39,6 +39,7 @@
 #include "cache.h"
 
 #include "cache_backend.h"
+#include "cache_pool.h"
 #include "vtcp.h"
 #include "vsa.h"
 #include "vtim.h"
@@ -56,7 +57,6 @@ struct tcp_pool {
 	struct lock		mtx;
 
 	struct waitfor		waitfor;
-	struct waiter		*waiter;
 	volatile double		timeout;
 
 	VTAILQ_HEAD(, vbc)	connlist;
@@ -166,7 +166,6 @@ VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6)
 	INIT_OBJ(&tp->waitfor, WAITFOR_MAGIC);
 	tp->waitfor.func = tcp_handle;
 	tp->waitfor.tmo = &tp->timeout;
-	tp->waiter = Waiter_New(&tp->waitfor);
 	return (tp);
 }
 
@@ -211,7 +210,6 @@ VBT_Rel(struct tcp_pool **tpp)
 	Lck_Delete(&tp->mtx);
 	AZ(tp->n_conn);
 	AZ(tp->n_kill);
-	Waiter_Destroy(&tp->waiter);
 
 	FREE_OBJ(tp);
 }
@@ -250,11 +248,12 @@ VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa)
  */
 
 void
-VBT_Recycle(struct tcp_pool *tp, struct vbc **vbcp)
+VBT_Recycle(const struct worker *wrk, struct tcp_pool *tp, struct vbc **vbcp)
 {
 	struct vbc *vbc;
 	int i = 0;
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC);
 	vbc = *vbcp;
 	*vbcp = NULL;
@@ -271,7 +270,8 @@ VBT_Recycle(struct tcp_pool *tp, struct vbc **vbcp)
 	vbc->waited->fd = vbc->fd;
 	vbc->waited->idle = VTIM_real();
 	vbc->state = VBC_STATE_AVAIL;
-	if (Wait_Enter(tp->waiter, vbc->waited)) {
+	vbc->waited->waitfor =  &tp->waitfor;
+	if (Wait_Enter(wrk->pool->waiter, vbc->waited)) {
 		VTCP_close(&vbc->fd);
 		memset(vbc, 0x33, sizeof *vbc);
 		free(vbc);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 76d47b8..2d55cba 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -482,6 +482,7 @@ SES_Wait(struct sess *sp)
 	sp->waited.fd = sp->fd;
 	sp->waited.ptr = sp;
 	sp->waited.idle = sp->t_idle;
+	sp->waited.waitfor = &pp->wf;
 	if (Wait_Enter(pp->waiter, &sp->waited))
 		SES_Delete(sp, SC_PIPE_OVERFLOW, NAN);
 }
@@ -587,5 +588,5 @@ SES_NewPool(struct pool *pp, unsigned pool_no)
 	INIT_OBJ(&pp->wf, WAITFOR_MAGIC);
 	pp->wf.func = ses_handle;
 	pp->wf.tmo = &cache_param->timeout_idle;
-	pp->waiter = Waiter_New(&pp->wf);
+	pp->waiter = Waiter_New();
 }
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index 4d2f794..7d23da5 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -51,7 +51,7 @@ waited_cmp(void *priv, const void *a, const void *b)
 	CAST_OBJ_NOTNULL(aa, a, WAITED_MAGIC);
 	CAST_OBJ_NOTNULL(bb, b, WAITED_MAGIC);
 
-	return (Wait_When(ww, aa) < Wait_When(ww, bb));
+	return (Wait_When(aa) < Wait_When(bb));
 }
 
 static void __match_proto__(binheap_update_t)
@@ -72,12 +72,12 @@ Wait_Call(const struct waiter *w,
 {
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
-	CHECK_OBJ_NOTNULL(w->waitfor, WAITFOR_MAGIC);
-	AN(w->waitfor->func);
+	CHECK_OBJ_NOTNULL(wp->waitfor, WAITFOR_MAGIC);
+	AN(wp->waitfor->func);
 	if (wp->idx != BINHEAP_NOIDX)
 		binheap_delete(w->heap, wp->idx);
 	assert(wp->idx == BINHEAP_NOIDX);
-	w->waitfor->func(wp, ev, now);
+	wp->waitfor->func(wp, ev, now);
 }
 
 /**********************************************************************/
@@ -105,7 +105,7 @@ Wait_HeapDue(const struct waiter *w, struct waited **wpp)
 	}
 	if (wpp != NULL)
 		*wpp = wp;
-	return(Wait_When(w, wp));
+	return(Wait_When(wp));
 }
 
 /**********************************************************************/
@@ -117,6 +117,7 @@ Wait_Enter(const struct waiter *w, struct waited *wp)
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 	assert(wp->fd > 0);			// stdin never comes here
+	CHECK_OBJ_NOTNULL(wp->waitfor, WAITFOR_MAGIC);
 	wp->idx = BINHEAP_NOIDX;
 	return (w->impl->enter(w->priv, wp));
 }
@@ -134,7 +135,7 @@ Waiter_GetName(void)
 }
 
 struct waiter *
-Waiter_New(struct waitfor *wf)
+Waiter_New(void)
 {
 	struct waiter *w;
 
@@ -149,7 +150,6 @@ Waiter_New(struct waitfor *wf)
 	INIT_OBJ(w, WAITER_MAGIC);
 	w->priv = (void*)(w + 1);
 	w->impl = waiter;
-	w->waitfor = wf;
 	VTAILQ_INIT(&w->waithead);
 	w->heap = binheap_new(w, waited_cmp, waited_update);
 
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index ea5ea22..90ba41d 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -148,7 +148,7 @@ vwk_enter(void *priv, struct waited *wp)
 	AZ(kevent(vwk->kq, &ke, 1, NULL, 0, NULL));
 
 	/* If the kqueue isn't due before our timeout, poke it via the pipe */
-	if (Wait_When(vwk->waiter, wp) < vwk->next)
+	if (Wait_When(wp) < vwk->next)
 		assert(write(vwk->pipe[1], "X", 1) == 1);
 
 	Lck_Unlock(&vwk->mtx);
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index 5c56f9b..471d9e7 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -183,7 +183,7 @@ vwp_main(void *priv)
 				break;
 			if (vwp->pollfd[i].revents)
 				v--;
-			then = Wait_When(vwp->waiter, wp);
+			then = Wait_When(wp);
 			if (then <= now) {
 				Wait_Call(vwp->waiter, wp, WAITER_TIMEOUT, now);
 				vwp_del(vwp, i);
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
index 7227fcd..3d301ea 100644
--- a/bin/varnishd/waiter/waiter.h
+++ b/bin/varnishd/waiter/waiter.h
@@ -67,12 +67,13 @@ struct waited {
 	int			fd;
 	unsigned		idx;
 	void			*ptr;
+	const struct waitfor	*waitfor;
 	double			idle;
 	VTAILQ_ENTRY(waited)	list;
 };
 
 /* cache_waiter.c */
 int Wait_Enter(const struct waiter *, struct waited *);
-struct waiter *Waiter_New(struct waitfor *);
+struct waiter *Waiter_New(void);
 void Waiter_Destroy(struct waiter **);
 const char *Waiter_GetName(void);
diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h
index 331e3ad..f4e5261 100644
--- a/bin/varnishd/waiter/waiter_priv.h
+++ b/bin/varnishd/waiter/waiter_priv.h
@@ -39,8 +39,6 @@ struct waiter {
 	VTAILQ_ENTRY(waiter)		list;
 	VTAILQ_HEAD(,waited)		waithead;
 
-	struct waitfor			*waitfor;
-
 	void				*priv;
 	struct binheap			*heap;
 };
@@ -61,19 +59,20 @@ struct waiter_impl {
 };
 
 static inline double
-Wait_Tmo(const struct waiter *w, const struct waited *wp)
+Wait_Tmo(const struct waited *wp)
 {
-	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	CHECK_OBJ_ORNULL(wp, WAITED_MAGIC);
-	CHECK_OBJ_NOTNULL(w->waitfor, WAITFOR_MAGIC);
-	AN(w->waitfor->tmo);
-	return (*w->waitfor->tmo);
+	CHECK_OBJ_NOTNULL(wp->waitfor, WAITFOR_MAGIC);
+	AN(wp->waitfor->tmo);
+	return (*wp->waitfor->tmo);
 }
 
 static inline double
-Wait_When(const struct waiter *w, const struct waited *wp)
+Wait_When(const struct waited *wp)
 {
-	return (Wait_Tmo(w, wp) + wp->idle);
+	CHECK_OBJ_ORNULL(wp, WAITED_MAGIC);
+	CHECK_OBJ_NOTNULL(wp->waitfor, WAITFOR_MAGIC);
+	return (wp->idle + *wp->waitfor->tmo);
 }
 
 void Wait_Call(const struct waiter *, struct waited *,



From phk at FreeBSD.org  Wed May 27 21:38:32 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 27 May 2015 23:38:32 +0200
Subject: [master] 8b5670d Disable epoll waiter for now.
Message-ID: <E1Yxj20-0002zF-J1@project.varnish-software.com>


commit 8b5670ddf0a300912d8df962807f07a8bd8e6fe0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 27 21:38:16 2015 +0000

    Disable epoll waiter for now.

diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c
index 18b3269..a8c036b 100644
--- a/bin/varnishd/waiter/mgt_waiter.c
+++ b/bin/varnishd/waiter/mgt_waiter.c
@@ -40,10 +40,10 @@ static const struct choice waiter_choice[] = {
     #if defined(HAVE_KQUEUE)
 	{ "kqueue",	&waiter_kqueue },
     #endif
+#if 0
     #if defined(HAVE_EPOLL_CTL)
 	{ "epoll",	&waiter_epoll },
     #endif
-#if 0
     #if defined(HAVE_PORT_CREATE)
 	{ "ports",	&waiter_ports },
     #endif



From phk at FreeBSD.org  Wed May 27 22:04:17 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 28 May 2015 00:04:17 +0200
Subject: [master] 427e00c Eliminate the list of struct waited,
 a simple count will do.
Message-ID: <E1YxjQv-0003EI-1p@project.varnish-software.com>


commit 427e00c9c6612d74b78076e564cee9488f7b1606
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 27 22:03:57 2015 +0000

    Eliminate the list of struct waited, a simple count will do.

diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index 90ba41d..baeed5c 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -54,10 +54,9 @@ struct vwk {
 	pthread_t		thread;
 	double			next;
 	int			pipe[2];
-
-	VTAILQ_HEAD(,waited)	list;
-	struct lock		mtx;
+	unsigned		nwaited;
 	int			die;
+	struct lock		mtx;
 };
 
 /*--------------------------------------------------------------------*/
@@ -116,13 +115,13 @@ vwk_thread(void *priv)
 				continue;
 			}
 			CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC);
-			VTAILQ_REMOVE(&vwk->list, wp, list);
+			vwk->nwaited--;
 			if (kp->flags & EV_EOF)
 				Wait_Call(w, wp, WAITER_REMCLOSE, now);
 			else
 				Wait_Call(w, wp, WAITER_ACTION, now);
 		}
-		if (VTAILQ_EMPTY(&vwk->list) && vwk->die)
+		if (vwk->nwaited == 0 && vwk->die)
 			break;
 	}
 	Lck_Unlock(&vwk->mtx);
@@ -143,7 +142,7 @@ vwk_enter(void *priv, struct waited *wp)
 	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
 	EV_SET(&ke, wp->fd, EVFILT_READ, EV_ADD|EV_ONESHOT, 0, 0, wp);
 	Lck_Lock(&vwk->mtx);
-	VTAILQ_INSERT_TAIL(&vwk->list, wp, list);
+	vwk->nwaited++;
 	Wait_HeapInsert(vwk->waiter, wp);
 	AZ(kevent(vwk->kq, &ke, 1, NULL, 0, NULL));
 
@@ -170,7 +169,6 @@ vwk_init(struct waiter *w)
 
 	vwk->kq = kqueue();
 	assert(vwk->kq >= 0);
-	VTAILQ_INIT(&vwk->list);
 	Lck_New(&vwk->mtx, lck_misc);
 	AZ(pipe(vwk->pipe));
 	EV_SET(&ke, vwk->pipe[0], EVFILT_READ, EV_ADD, 0, 0, vwk);
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
index 3d301ea..488c8e3 100644
--- a/bin/varnishd/waiter/waiter.h
+++ b/bin/varnishd/waiter/waiter.h
@@ -69,7 +69,6 @@ struct waited {
 	void			*ptr;
 	const struct waitfor	*waitfor;
 	double			idle;
-	VTAILQ_ENTRY(waited)	list;
 };
 
 /* cache_waiter.c */



From phk at FreeBSD.org  Wed May 27 22:09:10 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 28 May 2015 00:09:10 +0200
Subject: [master] c037d5b Rewrite the epoll waiter to the new reality.
Message-ID: <E1YxjVe-00035X-JV@project.varnish-software.com>


commit c037d5b21a345a3eead0be9daaee221523a20a81
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 27 22:08:53 2015 +0000

    Rewrite the epoll waiter to the new reality.

diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index bba3754..192c690 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -58,83 +58,85 @@ struct vwe {
 	int			epfd;
 	struct waiter		*waiter;
 	pthread_t		thread;
-	VTAILQ_HEAD(,waited)	list;
-	struct lock		mtx;
+	double			next;
+	int			pipe[2];
+	unsigned		nwaited;
 	int			die;
+	struct lock		mtx;
 };
 
-static void
-vwe_eev(struct vwe *vwe, const struct epoll_event *ep, double now)
-{
-	struct waited *wp;
-
-	AN(ep->data.ptr);
-	CAST_OBJ_NOTNULL(wp, ep->data.ptr, WAITED_MAGIC);
-	if (ep->events & EPOLLIN) {
-		Wait_Call(vwe->waiter, wp, WAITER_ACTION, now);
-	} else if (ep->events & EPOLLERR) {
-		Wait_Call(vwe->waiter, wp, WAITER_REMCLOSE, now);
-	} else if (ep->events & EPOLLHUP) {
-		Wait_Call(vwe->waiter, wp, WAITER_REMCLOSE, now);
-	} else if (ep->events & EPOLLRDHUP) {
-		Wait_Call(vwe->waiter, wp, WAITER_REMCLOSE, now);
-	}
-}
-
 /*--------------------------------------------------------------------*/
 
 static void *
 vwe_thread(void *priv)
 {
 	struct epoll_event ev[NEEV], *ep;
-	struct waited *wp, *wp2;
-	double now, idle, last_idle;
+	struct waited *wp;
+	struct waiter *w;
+	double now, then;
 	int i, n;
 	struct vwe *vwe;
-	VTAILQ_HEAD(,waited) tlist;
+	char c;
 
 	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
-
+	w = vwe->waiter;
+	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	THR_SetName("cache-epoll");
 
-	last_idle = 0.0;
+	now = VTIM_real();
+	Lck_Lock(&vwe->mtx);
 	while (1) {
-		i = floor(.3 * 1e3 * Wait_Tmo(vwe->waiter, NULL));
+		while (1) {
+			/*
+			 * XXX: We could avoid many syscalls here if we were
+			 * XXX: allowed to just close the fd's on timeout.
+			 */
+			then = Wait_HeapDue(w, &wp);
+			if (wp == NULL) {
+				vwe->next = now + 100;
+				break;
+			} else if (then > now) {
+				vwe->next = then;
+				break;
+			}
+			CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
+			AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL));
+			vwe->nwaited--;
+			Wait_Call(w, wp, WAITER_TIMEOUT, now);
+		}
+		then = vwe->next - now;
+		i = (int)floor(1e3 * then);
+		assert(i > 0);
+		Lck_Unlock(&vwe->mtx);
 		n = epoll_wait(vwe->epfd, ev, NEEV, i);
-		if (n < 0 && vwe->die)
-			break;
 		assert(n >= 0);
+		assert(n <= NEEV);
 		now = VTIM_real();
-		for (ep = ev, i = 0; i < n; i++, ep++) {
-			CAST_OBJ_NOTNULL(wp, ep->data.ptr, WAITED_MAGIC);
-			AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL));
-			Lck_Lock(&vwe->mtx);
-			VTAILQ_REMOVE(&vwe->list, wp, list);
-			Lck_Unlock(&vwe->mtx);
-			vwe_eev(vwe, ep, now);
-		}
-		idle = now - Wait_Tmo(vwe->waiter, NULL);
-		if (now - last_idle < .3 * Wait_Tmo(vwe->waiter, NULL))
-			continue;
-		last_idle = now;
-		VTAILQ_INIT(&tlist);
 		Lck_Lock(&vwe->mtx);
-		VTAILQ_FOREACH_SAFE(wp, &vwe->list, list, wp2) {
-			if (wp->idle > idle)
+		for (ep = ev, i = 0; i < n; i++, ep++) {
+			if (ep->data.ptr == vwe) {
+				assert(read(vwe->pipe[0], &c, 1) == 1);
 				continue;
-			VTAILQ_REMOVE(&vwe->list, wp, list);
-			VTAILQ_INSERT_TAIL(&tlist, wp, list);
+			}
+			CAST_OBJ_NOTNULL(wp, ep->data.ptr, WAITED_MAGIC);
 			AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL));
+			vwe->nwaited--;
+			if (ep->events & EPOLLIN)
+				Wait_Call(w, wp, WAITER_ACTION, now);
+			else if (ep->events & EPOLLERR)
+				Wait_Call(w, wp, WAITER_REMCLOSE, now);
+			else if (ep->events & EPOLLHUP)
+				Wait_Call(w, wp, WAITER_REMCLOSE, now);
+			else
+				Wait_Call(w, wp, WAITER_REMCLOSE, now);
 		}
-		Lck_Unlock(&vwe->mtx);
-		while(1) {
-			wp = VTAILQ_FIRST(&tlist);
-			if (wp == NULL)
-				break;
-			VTAILQ_REMOVE(&tlist, wp, list);
-			Wait_Call(vwe->waiter, wp, WAITER_TIMEOUT, now);
-		}
+		if (vwe->nwaited == 0 && vwe->die)
+			break;
 	}
+	Lck_Unlock(&vwe->mtx);
+	AZ(close(vwe->pipe[0]));
+	AZ(close(vwe->pipe[1]));
+	AZ(close(vwe->epfd));
 	return (NULL);
 }
 
@@ -150,8 +152,12 @@ vwe_enter(void *priv, struct waited *wp)
 	ee.events = EPOLLIN | EPOLLRDHUP;
 	ee.data.ptr = wp;
 	Lck_Lock(&vwe->mtx);
-	VTAILQ_INSERT_TAIL(&vwe->list, wp, list);
+	vwe->nwaited++;
+	Wait_HeapInsert(vwe->waiter, wp);
 	AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_ADD, wp->fd, &ee));
+	/* If the epoll isn't due before our timeout, poke it via the pipe */
+	if (Wait_When(wp) < vwe->next)
+		assert(write(vwe->pipe[1], "X", 1) == 1);
 	Lck_Unlock(&vwe->mtx);
 	return(0);
 }
@@ -162,6 +168,7 @@ static void __match_proto__(waiter_init_f)
 vwe_init(struct waiter *w)
 {
 	struct vwe *vwe;
+	struct epoll_event ee;
 
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	vwe = w->priv;
@@ -170,8 +177,11 @@ vwe_init(struct waiter *w)
 
 	vwe->epfd = epoll_create(1);
 	assert(vwe->epfd >= 0);
-	VTAILQ_INIT(&vwe->list);
 	Lck_New(&vwe->mtx, lck_misc);
+	AZ(pipe(vwe->pipe));
+	ee.events = EPOLLIN | EPOLLRDHUP;
+	ee.data.ptr = vwe;
+	AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_ADD, vwe->pipe[0], &ee));
 
 	AZ(pthread_create(&vwe->thread, NULL, vwe_thread, vwe));
 }
@@ -186,20 +196,12 @@ vwe_fini(struct waiter *w)
 {
 	struct vwe *vwe;
 	void *vp;
-	int i;
 
 	CAST_OBJ_NOTNULL(vwe, w->priv, VWE_MAGIC);
 
 	Lck_Lock(&vwe->mtx);
-	while (!VTAILQ_EMPTY(&vwe->list)) {
-		Lck_Unlock(&vwe->mtx);
-		(void)usleep(100000);
-		Lck_Lock(&vwe->mtx);
-	}
 	vwe->die = 1;
-	i = vwe->epfd;
-	vwe->epfd = -1;
-	AZ(close(i));
+	assert(write(vwe->pipe[1], "Y", 1) == 1);
 	Lck_Unlock(&vwe->mtx);
 	AZ(pthread_join(vwe->thread, &vp));
 	Lck_Delete(&vwe->mtx);
diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c
index a8c036b..18b3269 100644
--- a/bin/varnishd/waiter/mgt_waiter.c
+++ b/bin/varnishd/waiter/mgt_waiter.c
@@ -40,10 +40,10 @@ static const struct choice waiter_choice[] = {
     #if defined(HAVE_KQUEUE)
 	{ "kqueue",	&waiter_kqueue },
     #endif
-#if 0
     #if defined(HAVE_EPOLL_CTL)
 	{ "epoll",	&waiter_epoll },
     #endif
+#if 0
     #if defined(HAVE_PORT_CREATE)
 	{ "ports",	&waiter_ports },
     #endif



From nils.goroll at uplex.de  Thu May 28 08:33:16 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 28 May 2015 10:33:16 +0200
Subject: [master] b1b3c93 init jails when called with no arguments
Message-ID: <E1YxtFc-0001DD-J7@project.varnish-software.com>


commit b1b3c93d3727a3fdc936b4076dd8268350d075b7
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu May 28 10:33:04 2015 +0200

    init jails when called with no arguments

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 9aa5ef3..78a1ed3 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -509,6 +509,11 @@ main(int argc, char * const *argv)
 	init_params(cli);
 	cli_check(cli);
 
+	if (argc == 1) {
+		jailed++;
+		VJ_Init(NULL);
+	}
+
 	while ((o = getopt(argc, argv,
 	    "a:b:Cdf:Fh:i:j:l:M:n:P:p:r:S:s:T:t:VW:x:")) != -1) {
 		/*



From nils.goroll at uplex.de  Thu May 28 08:44:30 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 28 May 2015 10:44:30 +0200
Subject: [master] c3d7467 T_arg has a sensible default
Message-ID: <E1YxtQU-0002JA-PI@project.varnish-software.com>


commit c3d746755f90b0a1a13bfef4b4600da37168bcfc
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu May 28 10:44:14 2015 +0200

    T_arg has a sensible default

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 78a1ed3..16ef14b 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -654,9 +654,8 @@ main(int argc, char * const *argv)
 	if (b_arg != NULL && f_arg != NULL)
 		ARGV_ERR("Only one of -b or -f can be specified\n");
 
-	if (T_arg == NULL && d_flag == 0 && b_arg == NULL &&
-	    f_arg == NULL && M_arg == NULL)
-		ARGV_ERR("At least one of -d, -b, -f, -M or -T "
+	if (d_flag == 0 && b_arg == NULL && f_arg == NULL && M_arg == NULL)
+		ARGV_ERR("At least one of -d, -b, -f or -M "
 		    "must be specified\n");
 
 	if (S_arg != NULL && *S_arg == '\0') {



From nils.goroll at uplex.de  Thu May 28 10:44:07 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 28 May 2015 12:44:07 +0200
Subject: [master] a942313 minor docfix
Message-ID: <E1YxvIF-0004RK-EA@project.varnish-software.com>


commit a94231391712e2cee47f2fcfca54194a14540f7e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu May 28 12:43:59 2015 +0200

    minor docfix

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 91d977a..65181cb 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -61,8 +61,10 @@ OPTIONS
 -f config
 
   Use the specified VCL configuration file instead of the builtin
-  default.  See :ref:`vcl(7)` for details on VCL syntax. When no
-  configuration is supplied `varnishd` will not start the child process.
+  default.  See :ref:`vcl(7)` for details on VCL syntax.
+
+  When neither a -f nor a -b argument are given, `varnishd` will not
+  start the worker process but process cli commands.
 
 -F
 



From nils.goroll at uplex.de  Thu May 28 12:03:50 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 28 May 2015 14:03:50 +0200
Subject: [master] 98491be http is 80/tcp
Message-ID: <E1YxwXO-0004fh-RT@project.varnish-software.com>


commit 98491bef37b248a5bf345169e86de32b2d933855
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu May 28 13:32:14 2015 +0200

    http is 80/tcp
    
    Some systems cannot resolve service "http", so in commit
    8c7e459dc19542646bba5ca5789bdf99dc5724d0 we changed the
    hardcoded default service from "http" to "80".
    
    Before we complicate the implementation uselessly to first look
    up http and then fall back to 80, just docfix. Port 80 is probably
    _the_ one assignment with the least chance of ever changing.

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 65181cb..6494524 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -35,8 +35,8 @@ OPTIONS
   address can be a host name ("localhost"), an IPv4 dotted-quad
   ("127.0.0.1"), or an IPv6 address enclosed in square brackets
   ("[::1]"). If address is not specified, `varnishd` will listen on all
-  available IPv4 and IPv6 interfaces. If port is not specified, the
-  default HTTP port as listed in ``/etc/services`` is used.
+  available IPv4 and IPv6 interfaces. If port is not specified, port
+  80 (http) is used.
   An additional protocol type can be set for the listening socket with PROTO.
   Valid protocol types are: HTTP/1 (default), and PROXY.
   Multiple listening adresses can be specificed by using multiple -a arguments.



From nils.goroll at uplex.de  Thu May 28 12:03:50 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 28 May 2015 14:03:50 +0200
Subject: [master] b2065e8 listen to *:80 by default as documented in
 varnishd.rst
Message-ID: <E1YxwXO-0004fn-Sc@project.varnish-software.com>


commit b2065e8bc203ac9720d61cc1ee63dd749541d329
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu May 28 13:48:19 2015 +0200

    listen to *:80 by default as documented in varnishd.rst

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 16ef14b..eafdcb6 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -141,7 +141,8 @@ usage(void)
 #define FMT "    %-28s # %s\n"
 
 	fprintf(stderr, "usage: varnishd [options]\n");
-	fprintf(stderr, FMT, "-a address:port", "HTTP listen address and port");
+	fprintf(stderr, FMT, "-a address:port", "HTTP listen address and port"
+	    " (default: *:80)");
 	fprintf(stderr, FMT, "-b address:port", "backend address and port");
 	fprintf(stderr, FMT, "", "   -b <hostname_or_IP>");
 	fprintf(stderr, FMT, "", "   -b '<hostname_or_IP>:<port_or_service>'");
@@ -708,7 +709,9 @@ main(int argc, char * const *argv)
 		ARGV_ERR("-C only good with -b or -f\n");
 
 	if (VTAILQ_EMPTY(&heritage.socks))
-		ARGV_ERR("Need -a argument(s)\n");
+		MAC_Arg("*:80");
+
+	assert(! VTAILQ_EMPTY(&heritage.socks));
 
 	if (!d_flag) {
 		if (b_arg == NULL && f_arg == NULL) {



From nils.goroll at uplex.de  Thu May 28 12:03:50 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 28 May 2015 14:03:50 +0200
Subject: [master] a200018 we can run without any arguments
Message-ID: <E1YxwXO-0004fw-Td@project.varnish-software.com>


commit a200018f62e52aac6ffaa4ad8ae2b9603a1d0039
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu May 28 13:51:42 2015 +0200

    we can run without any arguments
    
    varnishd will (try to) to accept on the default port *:80 and listen to cli
    commands.
    
    A warning will get emitted:
    
    Warning: Neither -b nor -f given, won't start a worker child.
             Master process started, use varnishadm to control it.

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index eafdcb6..56c51b7 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -655,10 +655,6 @@ main(int argc, char * const *argv)
 	if (b_arg != NULL && f_arg != NULL)
 		ARGV_ERR("Only one of -b or -f can be specified\n");
 
-	if (d_flag == 0 && b_arg == NULL && f_arg == NULL && M_arg == NULL)
-		ARGV_ERR("At least one of -d, -b, -f or -M "
-		    "must be specified\n");
-
 	if (S_arg != NULL && *S_arg == '\0') {
 		fprintf(stderr,
 		    "Warning: Empty -S argument, no CLI authentication.\n");



From nils.goroll at uplex.de  Thu May 28 12:46:51 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 28 May 2015 14:46:51 +0200
Subject: [master] 74e2dd7 trying to guess phks intention
Message-ID: <E1YxxD1-0004Du-NK@project.varnish-software.com>


commit 74e2dd7161ccc1dcaee792a628587a1ff5452b84
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu May 28 14:46:22 2015 +0200

    trying to guess phks intention

diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index 192c690..47b036f 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -105,7 +105,7 @@ vwe_thread(void *priv)
 			Wait_Call(w, wp, WAITER_TIMEOUT, now);
 		}
 		then = vwe->next - now;
-		i = (int)floor(1e3 * then);
+		i = (int)ceil(1e3 * then);
 		assert(i > 0);
 		Lck_Unlock(&vwe->mtx);
 		n = epoll_wait(vwe->epfd, ev, NEEV, i);



From nils.goroll at uplex.de  Thu May 28 16:11:14 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 28 May 2015 18:11:14 +0200
Subject: [master] 28964cf Add back the ports waiter with the existing (struct
 waited *) passing though port_send
Message-ID: <E1Yy0Oo-00057I-H1@project.varnish-software.com>


commit 28964cf1cb071b53ef87a27ca060b25fb74a9ad7
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu May 28 17:42:50 2015 +0200

    Add back the ports waiter with the existing (struct waited *) passing though port_send
    
    See comment XXX questions to discuss with phk

diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index d02f302..f44959e 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2006 Varnish Software AS
  * Copyright (c) 2007 OmniTI Computer Consulting, Inc.
  * Copyright (c) 2007 Theo Schlossnagle
- * Copyright (c) 2010-2012 UPLEX, Nils Goroll
+ * Copyright (c) 2010-2015 UPLEX, Nils Goroll
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -29,7 +29,36 @@
  *
  */
 
-#if 0
+
+/*
+ * XXX questions to discuss with phk - is it really better
+ *
+ * - to share the binheap (requiring the mtx), which epoll / kq do now
+ *
+ * - than to send events to be entered through the events interface and keep the
+ *   binheap private to the waiter thread (which we still do here) ?
+ *
+ * at best, we can save two port syscalls, but not always:
+ *
+ * - if the waited event has a timeout earlier than the first element on the
+ *   binheap, we need to kick the waiter thread anyway
+ *
+ * - if the waiter thread is busy, it will get the passed waited event together
+ *   with other events
+ *
+ * on the other end we need to sync on the mtx to protect the binheap.  Solaris
+ * uses userland adaptive mutexes: if the thread holding the lock is running,
+ * spinlock, otherwise syscall.
+ *
+ * and the critical section for the mtx is basically "whenever not blocking in
+ * port_getn", which does not sound too good with respect to scalability.
+ *
+ * At any rate, we could save even more syscalls by increasing nevents
+ * (port_getn returns when nevents exist or the timeout is reached). This would
+ * increase our latency reacting on POLLIN events.
+ *
+ */
+
 #include "config.h"
 
 #if defined(HAVE_PORT_CREATE)
@@ -47,24 +76,24 @@
 #include "waiter/mgt_waiter.h"
 #include "vtim.h"
 
+// XXX replace with process.max-port-events bound to a sensible maximum
 #define MAX_EVENTS 256
 
 struct vws {
 	unsigned		magic;
 #define VWS_MAGIC		0x0b771473
 	struct waiter		*waiter;
-
 	pthread_t		thread;
+	double			next;
 	int			dport;
+	unsigned		nwaited;
+	int			die;
 };
 
 static inline void
 vws_add(struct vws *vws, int fd, void *data)
 {
-	/*
-	 * POLLIN should be all we need here
-	 *
-	 */
+	// POLLIN should be all we need here
 	AZ(port_associate(vws->dport, PORT_SOURCE_FD, fd, POLLIN, data));
 }
 
@@ -75,38 +104,29 @@ vws_del(struct vws *vws, int fd)
 }
 
 static inline void
-vws_port_ev(struct vws *vws, port_event_t *ev, double now) {
-	struct waited *sp;
+vws_port_ev(struct vws *vws, struct waiter *w, port_event_t *ev, double now) {
+	struct waited *wp;
 	if(ev->portev_source == PORT_SOURCE_USER) {
-		CAST_OBJ_NOTNULL(sp, ev->portev_user, WAITED_MAGIC);
-		assert(sp->fd >= 0);
-		VTAILQ_INSERT_TAIL(&vws->waiter->waithead, sp, list);
-		vws_add(vws, sp->fd, sp);
+		CAST_OBJ_NOTNULL(wp, ev->portev_user, WAITED_MAGIC);
+		assert(wp->fd >= 0);
+		vws->nwaited++;
+		Wait_HeapInsert(vws->waiter, wp);
+		vws_add(vws, wp->fd, wp);
 	} else {
 		assert(ev->portev_source == PORT_SOURCE_FD);
-		CAST_OBJ_NOTNULL(sp, ev->portev_user, WAITED_MAGIC);
-		assert(sp->fd >= 0);
-		if(ev->portev_events & POLLERR) {
-			vws_del(vws, sp->fd);
-			Wait_Handle(vws->waiter, sp, WAITER_REMCLOSE, now);
-			return;
-		}
-
+		CAST_OBJ_NOTNULL(wp, ev->portev_user, WAITED_MAGIC);
+		assert(wp->fd >= 0);
+		vws->nwaited--;
 		/*
-		 * note: the original man page for port_associate(3C) states:
-		 *
-		 *    When an event for a PORT_SOURCE_FD object is retrieved,
-		 *    the object no longer has an association with the port.
-		 *
-		 * This can be read along the lines of sparing the
-		 * port_dissociate after port_getn(), but in fact,
-		 * port_dissociate should be used
+		 * port_getn does not implicitly disassociate
 		 *
 		 * Ref: http://opensolaris.org/jive/thread.jspa?\
 		 *          threadID=129476&tstart=0
 		 */
-		vws_del(vws, sp->fd);
-		Wait_Handle(vws->waiter, sp, WAITER_ACTION, now);
+		vws_del(vws, wp->fd);
+		Wait_Call(w, wp, ev->portev_events & POLLERR ?
+		    WAITER_REMCLOSE : WAITER_ACTION,
+		    now);
 	}
 	return;
 }
@@ -114,53 +134,46 @@ vws_port_ev(struct vws *vws, port_event_t *ev, double now) {
 static void *
 vws_thread(void *priv)
 {
-	struct waited *sp;
+	struct waited *wp;
+	struct waiter *w;
 	struct vws *vws;
+	double now, then;
+	struct timespec ts;
+	const double	max_t  = 100.0;
+	port_event_t ev[MAX_EVENTS];
+	u_int nevents;
+	int ei, ret;
 
 	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
-	/*
-	 * timeouts:
-	 *
-	 * min_ts : Minimum timeout for port_getn
-	 * min_t  : ^ equivalent in floating point representation
-	 *
-	 * max_ts : Maximum timeout for port_getn
-	 * max_t  : ^ equivalent in floating point representation
-	 *
-	 * with (nevents == 1), we should always choose the correct port_getn
-	 * timeout to check session timeouts, so max is just a safety measure
-	 * (if this implementation is correct, it could be set to an "infinte"
-	 *  value)
-	 *
-	 * with (nevents > 1), min and max define the acceptable range for
-	 * - additional latency of keep-alive connections and
-	 * - additional tolerance for handling session timeouts
-	 *
-	 */
-	static struct timespec min_ts = {0L, 100L * 1000L * 1000L /*ns*/};
-	static double          min_t  = 0.1; /* 100    ms*/
-	static struct timespec max_ts = {1L, 0L};		/* 1 second */
-	static double	       max_t  = 1.0;			/* 1 second */
-
-	/* XXX: These should probably go in vws ? */
-	struct timespec ts;
-	struct timespec *timeout;
+	w = vws->waiter;
+	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
+	THR_SetName("cache-ports");
 
-	timeout = &max_ts;
+	now = VTIM_real();
 
-	while (!vws->waiter->dismantle) {
-		port_event_t ev[MAX_EVENTS];
-		u_int nevents;
-		int ei, ret;
-		double now, idle;
+	while (!vws->die) {
+		while (1) {
+			then = Wait_HeapDue(w, &wp);
+			if (wp == NULL) {
+				vws->next = now + max_t;
+				break;
+			} else if (then > now) {
+				vws->next = then;
+				break;
+			}
+			CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
+			vws_del(vws, wp->fd);
+			Wait_Call(w, wp, WAITER_TIMEOUT, now);
+		}
+		then = vws->next - now;
+		ts.tv_sec = (time_t)floor(then);
+		ts.tv_nsec = (long)(1e9 * (then - ts.tv_sec));
 
 		/*
-		 * XXX Do we want to scale this up dynamically to increase
-		 *     efficiency in high throughput situations? - would need to
-		 *     start with one to keep latency low at any rate
-		 *
-		 *     Note: when increasing nevents, we must lower min_ts
-		 *	     and max_ts
+		 * min number of events we accept.  could consider to scale up
+		 * for efficiency, but as we always get all waiting events up to
+		 * the maximum, we'd only optimize the idle case sacrificing
+		 * some latency
 		 */
 		nevents = 1;
 
@@ -177,12 +190,12 @@ vws_thread(void *priv)
 		 *
 		 */
 
-		ret = port_getn(vws->dport, ev, MAX_EVENTS, &nevents, timeout);
+		ret = port_getn(vws->dport, ev, MAX_EVENTS, &nevents, &ts);
 		now = VTIM_real();
 
 		if (ret < 0 && errno == EBADF) {
-			/* Our stop signal */
-			AN(vws->waiter->dismantle);
+			/* close on dport is our stop signal */
+			AN(vws->die);
 			break;
 		}
 
@@ -190,61 +203,21 @@ vws_thread(void *priv)
 			assert((errno == EINTR) || (errno == ETIME));
 
 		for (ei = 0; ei < nevents; ei++)
-			vws_port_ev(vws, ev + ei, now);
-
-		/* check for timeouts */
-		idle = now - *vws->waiter->tmo;
-
-		/*
-		 * This loop assumes that the oldest sessions are always at the
-		 * beginning of the list (which is the case if we guarantee to
-		 * enqueue at the tail only
-		 *
-		 */
-
-		for (;;) {
-			sp = VTAILQ_FIRST(&vws->waiter->waithead);
-			if (sp == NULL)
-				break;
-			if (sp->idle > idle) {
-				break;
-			}
-			vws_del(vws, sp->fd);
-			Wait_Handle(vws->waiter, sp, WAITER_TIMEOUT, now);
-		}
-
-		/*
-		 * Calculate the timeout for the next get_portn
-		 */
-
-		if (sp) {
-			double tmo = (sp->idle + *vws->waiter->tmo) - now;
-
-			if (tmo < min_t) {
-				timeout = &min_ts;
-			} else if (tmo > max_t) {
-				timeout = &max_ts;
-			} else {
-				ts = VTIM_timespec(tmo);
-				timeout = &ts;
-			}
-		} else {
-			timeout = &max_ts;
-		}
+			vws_port_ev(vws, w, &ev[ei], now);
 	}
-	return(0);
+	return NULL;
 }
 
 /*--------------------------------------------------------------------*/
 
 static int
-vws_pass(void *priv, struct waited *sp)
+vws_enter(void *priv, struct waited *wp)
 {
 	int r;
 	struct vws *vws;
 
 	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
-	r = port_send(vws->dport, 0, TRUST_ME(sp));
+	r = port_send(vws->dport, 0, TRUST_ME(wp));
 	if (r == -1 && errno == EAGAIN)
 		return (-1);
 	AZ(r);
@@ -277,6 +250,7 @@ vws_fini(struct waiter *w)
 	void *vp;
 
 	CAST_OBJ_NOTNULL(vws, w->priv, VWS_MAGIC);
+	vws->die = 1;
 	AZ(close(vws->dport));
 	AZ(pthread_join(vws->thread, &vp));
 }
@@ -287,9 +261,8 @@ const struct waiter_impl waiter_ports = {
 	.name =		"ports",
 	.init =		vws_init,
 	.fini =		vws_fini,
-	.pass =		vws_pass,
+	.enter =		vws_enter,
 	.size =		sizeof(struct vws),
 };
 
 #endif /* defined(HAVE_PORT_CREATE) */
-#endif
diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c
index 18b3269..ee7cef7 100644
--- a/bin/varnishd/waiter/mgt_waiter.c
+++ b/bin/varnishd/waiter/mgt_waiter.c
@@ -36,18 +36,16 @@
 #include "mgt/mgt.h"
 #include "waiter/mgt_waiter.h"
 
-static const struct choice waiter_choice[] = {
+const struct choice waiter_choice[] = {
     #if defined(HAVE_KQUEUE)
 	{ "kqueue",	&waiter_kqueue },
     #endif
     #if defined(HAVE_EPOLL_CTL)
 	{ "epoll",	&waiter_epoll },
     #endif
-#if 0
     #if defined(HAVE_PORT_CREATE)
 	{ "ports",	&waiter_ports },
     #endif
-#endif
 	{ "poll",	&waiter_poll },
 	{ NULL,		NULL}
 };



From phk at FreeBSD.org  Fri May 29 09:39:56 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 29 May 2015 11:39:56 +0200
Subject: [master] 7ed785a Put waiter.h in the right autocrap-category.
Message-ID: <E1YyGlg-0007x7-H6@project.varnish-software.com>


commit 7ed785a32edfd10aa62f9b900dd5824d578dfdb5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 29 09:39:39 2015 +0000

    Put waiter.h in the right autocrap-category.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 1b11581..e82770a 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -108,7 +108,6 @@ noinst_HEADERS = \
 	mgt/mgt_param.h \
 	storage/storage.h \
 	storage/storage_persistent.h \
-	waiter/waiter.h \
 	waiter/waiter_priv.h \
 	waiter/mgt_waiter.h
 
@@ -119,7 +118,8 @@ nobase_pkginclude_HEADERS = \
 	cache/cache_backend.h \
 	cache/cache_director.h \
 	common/common.h \
-	common/params.h
+	common/params.h \
+	waiter/waiter.h
 
 varnishd_CFLAGS = \
 	@PCRE_CFLAGS@ \



From nils.goroll at uplex.de  Fri May 29 11:38:52 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 29 May 2015 13:38:52 +0200
Subject: [master] 9123362 put the session's struct waited on the ws while
 waiting
Message-ID: <E1YyIcm-0003j5-M3@project.varnish-software.com>


commit 9123362ff7ea51a1e1d7e430c33b2efdb5c938f5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 29 13:38:48 2015 +0200

    put the session's struct waited on the ws while waiting

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 399fdcb..fa7ec7a 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -636,7 +636,7 @@ struct sess {
 
 	struct pool		*pool;
 
-	struct waited		waited;
+	struct waited		*waited;	// on ws while waiting
 
 	/* Session related fields ------------------------------------*/
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 2d55cba..e8824c8 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -431,8 +431,12 @@ ses_handle(struct waited *wp, enum wait_event ev, double now)
 
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 	CAST_OBJ_NOTNULL(sp, wp->ptr, SESS_MAGIC);
+	assert(sp->waited == wp);
+	wp->magic = 0;
+	wp = NULL;
+	sp->waited = NULL;
 
-	AZ(sp->ws->r);
+	WS_Release(sp->ws, 0);
 
 	switch (ev) {
 	case WAITER_TIMEOUT:
@@ -478,12 +482,19 @@ SES_Wait(struct sess *sp)
 		SES_Delete(sp, SC_REM_CLOSE, NAN);
 		return;
 	}
-	sp->waited.magic = WAITED_MAGIC;
-	sp->waited.fd = sp->fd;
-	sp->waited.ptr = sp;
-	sp->waited.idle = sp->t_idle;
-	sp->waited.waitfor = &pp->wf;
-	if (Wait_Enter(pp->waiter, &sp->waited))
+
+	AZ(sp->waited);
+	if (WS_Reserve(sp->ws, sizeof(struct waited))
+	    < sizeof(struct waited)) {
+		SES_Delete(sp, SC_OVERLOAD, NAN);
+	}
+	sp->waited = (void*)sp->ws->f;
+	INIT_OBJ(sp->waited, WAITED_MAGIC);
+	sp->waited->fd = sp->fd;
+	sp->waited->ptr = sp;
+	sp->waited->idle = sp->t_idle;
+	sp->waited->waitfor = &pp->wf;
+	if (Wait_Enter(pp->waiter, sp->waited))
 		SES_Delete(sp, SC_PIPE_OVERFLOW, NAN);
 }
 



From nils.goroll at uplex.de  Fri May 29 11:48:27 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 29 May 2015 13:48:27 +0200
Subject: [master] 9597fcd improve the note on concurrency
Message-ID: <E1YyIm3-0004je-Ss@project.varnish-software.com>


commit 9597fcd10fa947bb35edfc193f24e25b22f8429b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 29 13:47:32 2015 +0200

    improve the note on concurrency

diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index f44959e..20a636c 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -27,18 +27,20 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- */
-
-
-/*
- * XXX questions to discuss with phk - is it really better
+ * On concurrency:
+ *
+ * There are several options for the enter method to add an fd for the waiter
+ * thread to look after:
+ *
+ * - share the binheap (requiring a mutex) - implemented for epoll and kqueues
  *
- * - to share the binheap (requiring the mtx), which epoll / kq do now
+ * - send events to be entered through the events interface and keep the binheap
+ *   private to the waiter thread - implemented here.
  *
- * - than to send events to be entered through the events interface and keep the
- *   binheap private to the waiter thread (which we still do here) ?
+ * - some other message passing / mailbox
  *
- * at best, we can save two port syscalls, but not always:
+ * It has not yet been determined which option is best. In the best case, by
+ * sharing the binheap, we can save two port syscalls - but not always:
  *
  * - if the waited event has a timeout earlier than the first element on the
  *   binheap, we need to kick the waiter thread anyway



From nils.goroll at uplex.de  Fri May 29 12:03:06 2015
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 29 May 2015 14:03:06 +0200
Subject: [master] 020aaff save another 8 bytes in struct sess by removing the
 waited pointer
Message-ID: <E1YyJ0E-0006UU-3F@project.varnish-software.com>


commit 020aaff72da04293b4b6cbc6d49a31daad820e7d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 29 14:02:50 2015 +0200

    save another 8 bytes in struct sess by removing the waited pointer

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index fa7ec7a..f1cd990 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -636,8 +636,6 @@ struct sess {
 
 	struct pool		*pool;
 
-	struct waited		*waited;	// on ws while waiting
-
 	/* Session related fields ------------------------------------*/
 
 	struct ws		ws[1];
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index e8824c8..9820758 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -431,10 +431,9 @@ ses_handle(struct waited *wp, enum wait_event ev, double now)
 
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 	CAST_OBJ_NOTNULL(sp, wp->ptr, SESS_MAGIC);
-	assert(sp->waited == wp);
+	assert((void *)sp->ws->f == wp);
 	wp->magic = 0;
 	wp = NULL;
-	sp->waited = NULL;
 
 	WS_Release(sp->ws, 0);
 
@@ -470,6 +469,7 @@ void
 SES_Wait(struct sess *sp)
 {
 	struct pool *pp;
+	struct waited *wp;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	pp = sp->pool;
@@ -483,18 +483,20 @@ SES_Wait(struct sess *sp)
 		return;
 	}
 
-	AZ(sp->waited);
+	/*
+	 * put struct waited on the workspace
+	 */
 	if (WS_Reserve(sp->ws, sizeof(struct waited))
 	    < sizeof(struct waited)) {
 		SES_Delete(sp, SC_OVERLOAD, NAN);
 	}
-	sp->waited = (void*)sp->ws->f;
-	INIT_OBJ(sp->waited, WAITED_MAGIC);
-	sp->waited->fd = sp->fd;
-	sp->waited->ptr = sp;
-	sp->waited->idle = sp->t_idle;
-	sp->waited->waitfor = &pp->wf;
-	if (Wait_Enter(pp->waiter, sp->waited))
+	wp = (void*)sp->ws->f;
+	INIT_OBJ(wp, WAITED_MAGIC);
+	wp->fd = sp->fd;
+	wp->ptr = sp;
+	wp->idle = sp->t_idle;
+	wp->waitfor = &pp->wf;
+	if (Wait_Enter(pp->waiter, wp))
 		SES_Delete(sp, SC_PIPE_OVERFLOW, NAN);
 }
 



From phk at FreeBSD.org  Sat May 30 07:29:31 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 30 May 2015 09:29:31 +0200
Subject: [master] c4462c3 Add an overbroad #ifdef to catch the 32bit userland
 on 64bit SPARC config we have in Jenkins.
Message-ID: <E1YybD1-0002kG-5y@project.varnish-software.com>


commit c4462c333fe8d529ac180e56905633c926861039
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 30 07:28:51 2015 +0000

    Add an overbroad #ifdef to catch the 32bit userland on 64bit SPARC
    config we have in Jenkins.

diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index b268082..f487886 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -130,7 +130,12 @@ void VSM_common_ageupdate(const struct vsm_sc *sc);
  * Pointer aligment magic
  */
 
-#define PALGN	    (sizeof(void *) - 1)	/* size of alignment */
+#if defined(__sparc__)
+/* NB: Overbroad test for 32bit userland on 64bit SPARC cpus. */
+#  define PALGN	    (sizeof(double) - 1)	/* size of alignment */
+#else
+#  define PALGN	    (sizeof(void *) - 1)	/* size of alignment */
+#endif
 #define PAOK(p)	    (((uintptr_t)(p) & PALGN) == 0)	/* is aligned */
 #define PRNDDN(p)   ((uintptr_t)(p) & ~PALGN)		/* Round down */
 #define PRNDUP(p)   (((uintptr_t)(p) + PALGN) & ~PALGN)	/* Round up */



From phk at FreeBSD.org  Sat May 30 20:47:44 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 30 May 2015 22:47:44 +0200
Subject: [master] fa84722 Staticize.
Message-ID: <E1YynfU-0005fU-5e@project.varnish-software.com>


commit fa84722c49fd536dcc9d379f4e47634c8ea49d1c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 30 20:47:10 2015 +0000

    Staticize.

diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c
index ee7cef7..c0e450d 100644
--- a/bin/varnishd/waiter/mgt_waiter.c
+++ b/bin/varnishd/waiter/mgt_waiter.c
@@ -36,7 +36,7 @@
 #include "mgt/mgt.h"
 #include "waiter/mgt_waiter.h"
 
-const struct choice waiter_choice[] = {
+static const struct choice waiter_choice[] = {
     #if defined(HAVE_KQUEUE)
 	{ "kqueue",	&waiter_kqueue },
     #endif



From phk at FreeBSD.org  Sat May 30 20:47:44 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 30 May 2015 22:47:44 +0200
Subject: [master] 519fca1 Move more parameters over to the new table format.
Message-ID: <E1YynfU-0005fk-6t@project.varnish-software.com>


commit 519fca16c92dbef6c4e6bff99097fad3d506e666
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 30 20:47:22 2015 +0000

    Move more parameters over to the new table format.

diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index a20fe47..d623447 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -60,12 +60,14 @@ struct params {
 #define	ptyp_bool	unsigned
 #define	ptyp_double	double
 #define	ptyp_timeout	double
+#define	ptyp_uint	unsigned
 #define PARAM(nm, ty, mi, ma, de, un, fl, st, lt, fn) ptyp_##ty nm;
 #include <tbl/params.h>
 #undef PARAM
 #undef ptyp_bool
 #undef ptyp_double
 #undef ptyp_timeout
+#undef ptyp_uint
 
 	/* Unprivileged user / group */
 	char			*user;
@@ -130,9 +132,6 @@ struct params {
 	double			tcp_keepalive_intvl;
 #endif
 
-	/* Management hints */
-	unsigned		auto_restart;
-
 	/* Fetcher hints */
 	ssize_t			fetch_chunksize;
 	ssize_t			fetch_maxchunksize;
@@ -166,7 +165,6 @@ struct params {
 
 	/* Read timeouts for backend */
 	double			first_byte_timeout;
-	double			between_bytes_timeout;
 
 	/* CLI buffer size */
 	unsigned		cli_buffer;
@@ -177,13 +175,6 @@ struct params {
 	/* Acceptable clockskew with backends */
 	unsigned		clock_skew;
 
-	/* Get rid of duplicate bans */
-	unsigned		ban_dups;
-
-	double			ban_lurker_age;
-	double			ban_lurker_sleep;
-	unsigned		ban_lurker_batch;
-
 	unsigned		syslog_cli_traffic;
 
 	unsigned		http_range_support;
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index c97d3ff..f72714a 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -205,11 +205,6 @@ struct parspec mgt_parspec[] = {
 		"See setsockopt(2) under SO_SNDTIMEO for more information.",
 		DELAYED_EFFECT,
 		"60", "seconds" },
-	{ "auto_restart", tweak_bool, &mgt_param.auto_restart,
-		NULL, NULL,
-		"Restart child process automatically if it dies.",
-		0,
-		"on", "bool" },
 	{ "nuke_limit",
 		tweak_uint, &mgt_param.nuke_limit,
 		"0", NULL,
@@ -324,17 +319,6 @@ struct parspec mgt_parspec[] = {
 		"backend request. This parameter does not apply to pipe.",
 		0,
 		"60", "seconds" },
-	{ "between_bytes_timeout", tweak_timeout,
-		&mgt_param.between_bytes_timeout,
-		"0", NULL,
-		"Default timeout between bytes when receiving data from "
-		"backend. "
-		"We only wait for this many seconds between bytes "
-		"before giving up. A value of 0 means it will never time out. "
-		"VCL can override this default value for each backend request "
-		"and backend request. This parameter does not apply to pipe.",
-		0,
-		"60", "seconds" },
 	{ "clock_skew", tweak_uint, &mgt_param.clock_skew,
 		"0", NULL,
 		"How much clockskew we are willing to accept between the "
@@ -369,48 +353,11 @@ struct parspec mgt_parspec[] = {
 		"more sessions take a detour around the waiter.",
 		EXPERIMENTAL,
 		"0.050", "seconds" },
-	{ "ban_dups", tweak_bool, &mgt_param.ban_dups,
-		NULL, NULL,
-		"Eliminate older identical bans when new bans are created."
-		"  This test is CPU intensive and scales with the number and"
-		" complexity of active (non-Gone) bans.  If identical bans"
-		" are frequent, the amount of CPU needed to actually test "
-		" the bans will be similarly reduced.",
-		0,
-		"on", "bool" },
 	{ "syslog_cli_traffic", tweak_bool, &mgt_param.syslog_cli_traffic,
 		NULL, NULL,
 		"Log all CLI traffic to syslog(LOG_INFO).",
 		0,
 		"on", "bool" },
-	{ "ban_lurker_age", tweak_timeout,
-		&mgt_param.ban_lurker_age,
-		"0", NULL,
-		"The ban lurker does not process bans until they are this"
-		" old.  Right when a ban is added, the most frequently hit"
-		" objects will get tested against it as part of object"
-		" lookup.  This parameter prevents the ban-lurker from"
-		" kicking in, until the rush is over.",
-		0,
-		"60", "seconds" },
-	{ "ban_lurker_sleep", tweak_timeout,
-		&mgt_param.ban_lurker_sleep,
-		"0", NULL,
-		"The ban lurker thread sleeps between work batches, in order"
-		" to not monopolize CPU power."
-		"  When nothing is done, it sleeps a fraction of a second"
-		" before looking for new work to do.\n"
-		"A value of zero disables the ban lurker.",
-		0,
-		"0.01", "seconds" },
-	{ "ban_lurker_batch", tweak_uint,
-		&mgt_param.ban_lurker_batch,
-		"1", NULL,
-		"How many objects the ban lurker examines before taking a"
-		" ban_lurker_sleep.  Use this to pace the ban lurker so it"
-		" does not eat too much CPU.",
-		0,
-		"1000", "" },
 	{ "http_range_support", tweak_bool, &mgt_param.http_range_support,
 		NULL, NULL,
 		"Enable support for HTTP Range headers.",
diff --git a/include/tbl/params.h b/include/tbl/params.h
index 12afdaa..49fc7d0 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -97,108 +97,107 @@ PARAM(
 	/* func */	NULL
 )
 
-/**********************************************************************/
-#if 0 /* NOT YET */
 PARAM(
 	/* name */	auto_restart,
-	/* tweak */	tweak_bool,
-	/* var */	auto_restart,
-	/* min */	none,
-	/* max */	none,
-	/* default */	on,
-	/* units */	bool,
-	/* flags */	00,
+	/* typ */	bool,
+	/* min */	NULL,
+	/* max */	NULL,
+	/* default */	"on",
+	/* units */	"bool",
+	/* flags */	0,
 	/* s-text */
-	"Restart child process automatically if it dies.\n",
+	"Automatically restart the child/worker process if it dies.",
 	/* l-text */	"",
 	/* func */	NULL
 )
+
 PARAM(
 	/* name */	ban_dups,
-	/* tweak */	tweak_bool,
-	/* var */	ban_dups,
-	/* min */	none,
-	/* max */	none,
-	/* default */	on,
-	/* units */	bool,
-	/* flags */	00,
+	/* typ */	bool,
+	/* min */	NULL,
+	/* max */	NULL,
+	/* default */	"on",
+	/* units */	"bool",
+	/* flags */	0,
 	/* s-text */
-	"Eliminate older identical bans when new bans are created.  This "
-	"test is CPU intensive and scales with the number and complexity "
-	"of active (non-Gone) bans.  If identical bans are frequent, the "
-	"amount of CPU needed to actually test  the bans will be similarly "
-	"reduced.\n",
+	"Eliminate older identical bans when a new ban is added.  This saves "
+	"CPU cycles by not comparing objects to identical bans.\n"
+	"This is a waste of time if you have many bans which are never "
+	"identical.",
 	/* l-text */	"",
 	/* func */	NULL
 )
+
 PARAM(
 	/* name */	ban_lurker_age,
-	/* tweak */	tweak_timeout,
-	/* var */	ban_lurker_age,
-	/* min */	0.000,
-	/* max */	none,
-	/* default */	60.000,
-	/* units */	seconds,
-	/* flags */	00,
+	/* tweak */	timeout,
+	/* min */	"0",
+	/* max */	NULL,
+	/* default */	"60",
+	/* units */	"seconds",
+	/* flags */	0,
 	/* s-text */
-	"The ban lurker does not process bans until they are this old.  "
-	"Right when a ban is added, the most frequently hit objects will "
+	"The ban lurker only process bans when they are this old.  "
+	"When a ban is added, the most frequently hit objects will "
 	"get tested against it as part of object lookup.  This parameter "
-	"prevents the ban-lurker from kicking in, until the rush is over.\n",
+	"prevents the ban-lurker from kicking in, until the rush is over.",
 	/* l-text */	"",
 	/* func */	NULL
 )
+
 PARAM(
 	/* name */	ban_lurker_batch,
-	/* tweak */	tweak_uint,
-	/* var */	ban_lurker_batch,
-	/* min */	1,
-	/* max */	none,
-	/* default */	1000,
-	/* units */	,
-	/* flags */	00,
+	/* tweak */	uint,
+	/* min */	"1",
+	/* max */	NULL,
+	/* default */	"1000",
+	/* units */	NULL,
+	/* flags */	0,
 	/* s-text */
-	"How many objects the ban lurker examines before taking a "
-	"ban_lurker_sleep.  Use this to pace the ban lurker so it does not "
-	"eat too much CPU.\n",
+	"The ban lurker slees ${ban_lurker_sleep} after examining this "
+	"many objects.  Use this to pace the ban-lurker if it eats too "
+	"many resources.",
 	/* l-text */	"",
 	/* func */	NULL
 )
+
 PARAM(
 	/* name */	ban_lurker_sleep,
-	/* tweak */	tweak_timeout,
-	/* var */	ban_lurker_sleep,
-	/* min */	0.000,
-	/* max */	none,
-	/* default */	0.010,
-	/* units */	seconds,
-	/* flags */	00,
+	/* tweak */	timeout,
+	/* min */	"0",
+	/* max */	NULL,
+	/* default */	"0.010",
+	/* units */	"seconds",
+	/* flags */	0,
 	/* s-text */
-	"The ban lurker thread sleeps between work batches, in order to "
-	"not monopolize CPU power.  When nothing is done, it sleeps a "
-	"fraction of a second before looking for new work to do.\n"
-	"A value of zero disables the ban lurker.\n",
+	"How long the ban lurker sleeps after examining ${ban_lurker_batch} "
+	"objects.\n"
+	"A value of zero will disable the ban lurker entirely.",
 	/* l-text */	"",
 	/* func */	NULL
 )
+
 PARAM(
 	/* name */	between_bytes_timeout,
-	/* tweak */	tweak_timeout,
-	/* var */	between_bytes_timeout,
-	/* min */	0.000,
-	/* max */	none,
-	/* default */	60.000,
-	/* units */	seconds,
-	/* flags */	00,
+	/* tweak */	timeout,
+	/* min */	"0",
+	/* max */	NULL,
+	/* default */	"60",
+	/* units */	"seconds",
+	/* flags */	0,
 	/* s-text */
-	"Default timeout between bytes when receiving data from backend. "
-	"We only wait for this many seconds between bytes before giving "
-	"up. A value of 0 means it will never time out. VCL can override "
-	"this default value for each backend request and backend request. "
-	"This parameter does not apply to pipe.\n",
+	"We only wait for this many seconds between bytes received from "
+	"the backend before giving up the fetch.\n"
+	"A value of zero means never give up.\n"
+	"VCL values, per backend or per backend request take precedence.\n"
+	"This parameter does not apply to pipe'ed requests.",
 	/* l-text */	"",
 	/* func */	NULL
 )
+
+/**********************************************************************/
+#if 0 /* NOT YET */
+
 #if 0
 PARAM(
 	/* name */	busyobj_worker_cache,



From phk at FreeBSD.org  Sat May 30 20:59:25 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 30 May 2015 22:59:25 +0200
Subject: [master] ec38fd3 Remove the busyobj_worker_cache parameter,
 which has defaulted to "off" for a very long time and never been
 shown to actually improve performance.
Message-ID: <E1Yynqn-0006q3-0v@project.varnish-software.com>


commit ec38fd337091cd5b989f02633dc0596cb2799cee
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 30 20:57:26 2015 +0000

    Remove the busyobj_worker_cache parameter, which has defaulted to "off"
    for a very long time and never been shown to actually improve performance.
    
    These days we pick busyobj out of a memory pool which gives us much better
    control of the amount of memory sitting idle waiting to become a busyobj.
    
    In case a single mempool not scale well enough, we make a mempool per
    thread pool to scale horizontally.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index f1cd990..a2d1441 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -328,7 +328,6 @@ struct worker {
 	struct objhead		*nobjhead;
 	struct objcore		*nobjcore;
 	struct waitinglist	*nwaitinglist;
-	struct busyobj		*nbo;
 	void			*nhashpriv;
 	struct dstat		stats[1];
 	struct vsl_log		*vsl;		// borrowed from req/bo
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 2ba43fd..2ac9006 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -92,21 +92,14 @@ VBO_Free(struct busyobj **bop)
 struct busyobj *
 VBO_GetBusyObj(struct worker *wrk, const struct req *req)
 {
-	struct busyobj *bo = NULL;
+	struct busyobj *bo;
 	uint16_t nhttp;
 	unsigned sz;
 	char *p;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	if (wrk->nbo != NULL) {
-		bo = wrk->nbo;
-		wrk->nbo = NULL;
-	}
-
-	if (bo == NULL)
-		bo = vbo_New();
-
+	bo = vbo_New();
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	AZ(bo->refcount);
 
@@ -222,10 +215,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 	memset(&bo->refcount, 0,
 	    sizeof *bo - offsetof(struct busyobj, refcount));
 
-	if (cache_param->bo_cache && wrk != NULL && wrk->nbo == NULL)
-		wrk->nbo = bo;
-	else
-		VBO_Free(&bo);
+	VBO_Free(&bo);
 }
 
 void
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index e95b1eb..ee8bd5e 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -125,8 +125,6 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace)
 	if (w->vcl != NULL)
 		VCL_Rel(&w->vcl);
 	AZ(pthread_cond_destroy(&w->cond));
-	if (w->nbo != NULL)
-		VBO_Free(&w->nbo);
 	HSH_Cleanup(w);
 	Pool_Sumstat(w);
 }
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index d623447..c8cd3a6 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -194,8 +194,6 @@ struct params {
 
 	struct vre_limits	vre_limits;
 
-	unsigned		bo_cache;
-
 	/* Install a SIGSEGV handler */
 	unsigned		sigsegv_handler;
 
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index f72714a..60a0c4e 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -506,15 +506,6 @@ struct parspec mgt_parspec[] = {
 		MUST_RESTART,
 		"1M", "bytes"},
 
-	{ "busyobj_worker_cache", tweak_bool,
-		&mgt_param.bo_cache,
-		NULL, NULL,
-		"Cache free busyobj per worker thread. "
-		"Disable this if you have very high hitrates and want "
-		"to save the memory of one busyobj per worker thread.",
-		0,
-		"off", "bool"},
-
 	{ "pool_req", tweak_poolparam, &mgt_param.req_pool,
 		NULL, NULL,
 		"Parameters for per worker pool request memory pool.\n"
diff --git a/include/tbl/params.h b/include/tbl/params.h
index 49fc7d0..1581374 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -198,43 +198,6 @@ PARAM(
 /**********************************************************************/
 #if 0 /* NOT YET */
 
-#if 0
-PARAM(
-	/* name */	busyobj_worker_cache,
-	/* tweak */	tweak_bool,
-	/* var */	busyobj_worker_cache,
-	/* min */	none,
-	/* max */	none,
-	/* default */	off,
-	/* units */	bool,
-	/* flags */	00,
-	/* s-text */
-	"Cache free busyobj per worker thread. Disable this if you have "
-	"very high hitrates and want to save the memory of one busyobj per "
-	"worker thread.\n",
-	/* l-text */	"",
-	/* func */	NULL
-)
-#endif
-#if 0
-PARAM(
-	/* name */	cc_command,
-	/* tweak */	tweak_string,
-	/* var */	cc_command,
-	/* min */	none,
-	/* max */	none,
-	/* default */	"exec clang -std=gnu99 -g -O2 -Wall -Werror -Wno-error=unused-result  \t-Werror \t-Wall \t-Wno-format-y2k \t-W \t-Wstrict-prototypes \t-Wmissing-prototypes \t-Wpointer-arith \t-Wreturn-type \t-Wcast-qual \t-Wwrite-strings \t-Wswitch \t-Wshadow \t-Wunused-parameter \t-Wcast-align \t-Wchar-subscripts \t-Wnested-externs \t-Wextra \t-Wno-sign-compare  -fstack-protector -Wno-pointer-sign -Wno-address -Wno-missing-field-initializers -D_THREAD_SAFE -pthread -fpic -shared -Wl,-x -o %o %s",
-	/* units */	(null),
-	/* flags */	0| MUST_RELOAD,
-	/* s-text */
-	"Command used for compiling the C source code to a dlopen(3) "
-	"loadable object.  Any occurrence of %s in the string will be "
-	"replaced with the source file name, and %o will be replaced with "
-	"the output file name.\n",
-	/* l-text */	"",
-	/* func */	NULL
-)
-#endif
 PARAM(
 	/* name */	cli_buffer,
 	/* tweak */	tweak_bytes_u,



From phk at FreeBSD.org  Sat May 30 21:22:47 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 30 May 2015 23:22:47 +0200
Subject: [master] 0710944 Staticize.
Message-ID: <E1YyoDP-00016J-Ud@project.varnish-software.com>


commit 07109443e31bec07c1bf497b137f7c2bfb35b2c6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 30 21:20:18 2015 +0000

    Staticize.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a2d1441..0265717 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -699,7 +699,6 @@ double BAN_Time(const struct ban *ban);
 void VBO_Init(void);
 struct busyobj *VBO_GetBusyObj(struct worker *, const struct req *);
 void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
-void VBO_Free(struct busyobj **vbo);
 void VBO_extend(struct busyobj *, ssize_t);
 ssize_t VBO_waitlen(struct worker *, struct busyobj *, ssize_t l);
 void VBO_setstate(struct busyobj *bo, enum busyobj_state_e next);
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 2ac9006..d899378 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -73,8 +73,8 @@ vbo_New(void)
 	return (bo);
 }
 
-void
-VBO_Free(struct busyobj **bop)
+static void
+vbo_Free(struct busyobj **bop)
 {
 	struct busyobj *bo;
 
@@ -215,7 +215,7 @@ VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 	memset(&bo->refcount, 0,
 	    sizeof *bo - offsetof(struct busyobj, refcount));
 
-	VBO_Free(&bo);
+	vbo_Free(&bo);
 }
 
 void



From phk at FreeBSD.org  Sat May 30 23:14:34 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 31 May 2015 01:14:34 +0200
Subject: [master] e93ea04 Add a facility to schedule at task on the thread
 pools in round-robin order.
Message-ID: <E1Yypxa-0004eN-UP@project.varnish-software.com>


commit e93ea04a93d5b3136d96791d88d4b4698caa41e3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat May 30 23:13:38 2015 +0000

    Add a facility to schedule at task on the thread pools in round-robin
    order.
    
    Use it to health poke backends, rather than wasting a thread per
    backend+poll.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 0265717..3c38a04 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -953,6 +953,7 @@ int Pool_Task_Arg(struct worker *, task_func_t *,
 void Pool_Sumstat(struct worker *w);
 int Pool_TrySumstat(struct worker *wrk);
 void Pool_PurgeStat(unsigned nobj);
+int Pool_Task_Any(struct pool_task *task, enum pool_how how);
 
 #define V1L_IsReleased(w)	((w)->v1l == NULL)
 void V1L_Chunked(const struct worker *w);
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index e84ff06..2df74c2 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -352,10 +352,10 @@ VRT_event_vbe(VRT_CTX, enum vcl_event_e ev, const struct director *d,
 	}
 
 	if (be->probe != NULL && ev == VCL_EVENT_WARM)
-		VBP_Control(be, 0);
+		VBP_Control(be, 1);
 
 	if (be->probe != NULL && ev == VCL_EVENT_COLD)
-		VBP_Control(be, 1);
+		VBP_Control(be, 0);
 
 	if (ev == VCL_EVENT_COLD) {
 		VSM_Free(be->vsc);
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 11f14bd..d7c9aac 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -114,6 +114,7 @@ void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p,
 void VBP_Remove(struct backend *b);
 void VBP_Control(const struct backend *b, int stop);
 void VBP_Status(struct cli *cli, const struct backend *, int details);
+void VBP_Init(void);
 
 struct tcp_pool *VBT_Ref(const struct suckaddr *ip4,
     const struct suckaddr *ip6);
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index 2a9a863..8928d38 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -41,6 +41,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+#include "binary_heap.h"
+
 #include "cache.h"
 
 #include "cache_backend.h"
@@ -57,9 +59,6 @@ struct vbp_target {
 	unsigned			magic;
 #define VBP_TARGET_MAGIC		0x6b7cb656
 
-	struct lock			mtx;
-	int				disable;
-	int				stop;
 	struct backend			*backend;
 
 	struct tcp_pool			*tcp_pool;
@@ -81,12 +80,15 @@ struct vbp_target {
 	double				avg;
 	double				rate;
 
-	VTAILQ_ENTRY(vbp_target)	list;
-	pthread_t			thread;
+	double				due;
+	int				running;
+	int				heap_idx;
+	struct pool_task		task;
 };
 
-static VTAILQ_HEAD(, vbp_target)	vbp_list =
-    VTAILQ_HEAD_INITIALIZER(vbp_list);
+static struct lock			vbp_mtx;
+static pthread_cond_t			vbp_cond;
+static struct binheap			*vbp_heap;
 
 /*--------------------------------------------------------------------
  * Poke one backend, once, but possibly at both IPv4 and IPv6 addresses.
@@ -238,7 +240,7 @@ vbp_has_poked(struct vbp_target *vt)
 	}
 	vt->good = j;
 
-	Lck_Lock(&vt->mtx);
+	Lck_Lock(&vbp_mtx);
 	if (vt->backend != NULL) {
 		if (vt->good >= vt->probe.threshold) {
 			if (vt->backend->healthy)
@@ -260,48 +262,85 @@ vbp_has_poked(struct vbp_target *vt)
 		    vt->backend->display_name, logmsg, bits,
 		    vt->good, vt->probe.threshold, vt->probe.window,
 		    vt->last, vt->avg, vt->resp_buf);
-		if (!vt->disable) {
-			AN(vt->backend->vsc);
+		if (vt->backend != NULL && vt->backend->vsc != NULL)
 			vt->backend->vsc->happy = vt->happy;
-		}
 	}
-	Lck_Unlock(&vt->mtx);
+	Lck_Unlock(&vbp_mtx);
 }
 
 /*--------------------------------------------------------------------
- * One thread per backend to be poked.
  */
 
-static void *
-vbp_wrk_poll_backend(void *priv)
+static void __match_proto__(task_func_t)
+vbp_task(struct worker *wrk, void *priv)
 {
 	struct vbp_target *vt;
 
-	THR_SetName("backend poll");
-
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(vt, priv, VBP_TARGET_MAGIC);
 
-	while (!vt->stop) {
-		AN(vt->req);
-		assert(vt->req_len > 0);
+	AN(vt->req);
+	assert(vt->req_len > 0);
 
-		if (!vt->disable) {
-			vbp_start_poke(vt);
-			vbp_poke(vt);
-			vbp_has_poked(vt);
-		}
+	vbp_start_poke(vt);
+	vbp_poke(vt);
+	vbp_has_poked(vt);
+
+	Lck_Lock(&vbp_mtx);
+	if (vt->running < 0) {
+		VBT_Rel(&vt->tcp_pool);
+		free(vt->req);
+		FREE_OBJ(vt);
+	} else {
+		vt->running = 0;
+	}
+	Lck_Unlock(&vbp_mtx);
+}
+/*--------------------------------------------------------------------
+ */
+
+static void * __match_proto__()
+vbp_thread(struct worker *wrk, void *priv)
+{
+	double now, nxt;
+	struct vbp_target *vt;
 
-		if (!vt->stop)
-			VTIM_sleep(vt->probe.interval);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AZ(priv);
+	while (1) {
+		Lck_Lock(&vbp_mtx);
+		while (1) {
+			now = VTIM_real();
+			vt = binheap_root(vbp_heap);
+			if (vt == NULL) {
+				nxt = 8.192 + now;
+			} else if (vt->due > now) {
+				nxt = vt->due;
+				vt = NULL;
+			} else {
+				binheap_delete(vbp_heap, vt->heap_idx);
+				vt->running = 1;
+				vt->due = now + vt->probe.interval;
+				binheap_insert(vbp_heap, vt);
+				nxt = 0.0;
+				break;
+			}
+			(void)Lck_CondWait(&vbp_cond, &vbp_mtx, nxt);
+		}
+		Lck_Unlock(&vbp_mtx);
+		vt->task.func = vbp_task;
+		vt->task.priv = vt;
+
+		if (Pool_Task_Any(&vt->task, POOL_QUEUE_FRONT)) {
+			Lck_Lock(&vbp_mtx);
+			vt->running = 0;
+			Lck_Unlock(&vbp_mtx);
+			// XXX: ehh... ?
+		}
 	}
-	Lck_Delete(&vt->mtx);
-	VTAILQ_REMOVE(&vbp_list, vt, list);
-	VBT_Rel(&vt->tcp_pool);
-	free(vt->req);
-	FREE_OBJ(vt);
-	return (NULL);
 }
 
+
 /*--------------------------------------------------------------------
  * Cli functions
  */
@@ -420,7 +459,7 @@ vbp_set_defaults(struct vbp_target *vt)
  */
 
 void
-VBP_Control(const struct backend *be, int stop)
+VBP_Control(const struct backend *be, int enable)
 {
 	struct vbp_target *vt;
 
@@ -429,17 +468,18 @@ VBP_Control(const struct backend *be, int stop)
 	vt = be->probe;
 	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
 
-VSL(SLT_Debug, 0, "VBP_CONTROL %d", stop);
-	Lck_Lock(&vt->mtx);
-	if (vt->disable == -1 && !stop) {
-		vt->disable = stop;
-		AZ(pthread_create(&vt->thread, NULL, vbp_wrk_poll_backend, vt));
-		AZ(pthread_detach(vt->thread));
+VSL(SLT_Debug, 0, "VBP_CONTROL %d", enable);
+	Lck_Lock(&vbp_mtx);
+	if (enable) {
+		assert(vt->heap_idx == BINHEAP_NOIDX);
+		vt->due = VTIM_real();
+		binheap_insert(vbp_heap, vt);
+		AZ(pthread_cond_signal(&vbp_cond));
 	} else {
-		assert(vt->disable != -1);
-		vt->disable = stop;
+		assert(vt->heap_idx != BINHEAP_NOIDX);
+		binheap_delete(vbp_heap, vt->heap_idx);
 	}
-	Lck_Unlock(&vt->mtx);
+	Lck_Unlock(&vbp_mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -461,14 +501,12 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *p,
 
 	ALLOC_OBJ(vt, VBP_TARGET_MAGIC);
 	XXXAN(vt);
-	VTAILQ_INSERT_TAIL(&vbp_list, vt, list);
-	Lck_New(&vt->mtx, lck_backend);
-	vt->disable = -1;
 
 	vt->tcp_pool = VBT_Ref(b->ipv4, b->ipv6);
 	AN(vt->tcp_pool);
 
 	vt->probe = *p;
+	vt->backend = b;
 
 	vbp_set_defaults(vt);
 	vbp_build_req(vt, hosthdr);
@@ -480,7 +518,6 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *p,
 		vt->happy |= 1;
 		vbp_has_poked(vt);
 	}
-	vt->backend = b;
 	b->probe = vt;
 	vbp_has_poked(vt);
 }
@@ -495,11 +532,57 @@ VBP_Remove(struct backend *be)
 	vt = be->probe;
 	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
 
-	Lck_Lock(&vt->mtx);
-	vt->stop = 1;
-	vt->backend = NULL;
-	Lck_Unlock(&vt->mtx);
-
+	Lck_Lock(&vbp_mtx);
 	be->healthy = 1;
 	be->probe = NULL;
+	vt->backend = NULL;
+	if (vt->running) {
+		vt->running = -1;
+		vt = NULL;
+	}
+	Lck_Unlock(&vbp_mtx);
+	if (vt != NULL) {
+		VBT_Rel(&vt->tcp_pool);
+		free(vt->req);
+		FREE_OBJ(vt);
+	}
+}
+/*--------------------------------------------------------------------
+ */
+
+static int __match_proto__(binheap_cmp_t)
+vbp_cmp(void *priv, const void *a, const void *b)
+{
+	const struct vbp_target *aa, *bb;
+
+	AZ(priv);
+	CAST_OBJ_NOTNULL(aa, a, VBP_TARGET_MAGIC);
+	CAST_OBJ_NOTNULL(bb, b, VBP_TARGET_MAGIC);
+
+	return (aa->due < bb->due);
+}
+
+static void __match_proto__(binheap_update_t)
+vbp_update(void *priv, void *p, unsigned u)
+{
+	struct vbp_target *vt;
+
+	AZ(priv);
+	CAST_OBJ_NOTNULL(vt, p, VBP_TARGET_MAGIC);
+	vt->heap_idx = u;
+}
+
+/*--------------------------------------------------------------------
+ */
+
+void
+VBP_Init(void)
+{
+	pthread_t thr;
+
+	Lck_New(&vbp_mtx, lck_backend);
+	vbp_heap = binheap_new(NULL, vbp_cmp, vbp_update);
+	AN(vbp_heap);
+	AZ(pthread_cond_init(&vbp_cond, NULL));
+	WRK_BgThread(&thr, "Backend poller", vbp_thread, NULL);
 }
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 00db49d..4c5370f 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -33,6 +33,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "cache_backend.h"
 #include "common/heritage.h"
 
 #include "vcli_priv.h"
@@ -217,6 +218,7 @@ child_main(void)
 	HTTP_Init();
 
 	VBO_Init();
+	VBP_Init();
 	VBE_InitCfg();
 	Pool_Init();
 	V1P_Init();
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 628b18c..56f11bf 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -45,6 +45,7 @@ static pthread_t		thr_pool_herder;
 
 static struct lock		wstat_mtx;
 struct lock			pool_mtx;
+static VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
 
 /*--------------------------------------------------------------------
  * Summing of stats into global stats counters
@@ -86,6 +87,28 @@ Pool_TrySumstat(struct worker *wrk)
 }
 
 /*--------------------------------------------------------------------
+ * Facility for scheduling a task on any convenient pool.
+ */
+
+int
+Pool_Task_Any(struct pool_task *task, enum pool_how how)
+{
+	struct pool *pp;
+
+	Lck_Lock(&pool_mtx);
+	pp = VTAILQ_FIRST(&pools);
+	if (pp != NULL) {
+		VTAILQ_REMOVE(&pools, pp, list);
+		VTAILQ_INSERT_TAIL(&pools, pp, list);
+	}
+	Lck_Unlock(&pool_mtx);
+	if (pp == NULL)
+		return (-1);
+	// NB: When we remove pools, is there a race here ?
+	return (Pool_Task(pp, task, how));
+}
+
+/*--------------------------------------------------------------------
  * Helper function to update stats for purges under lock
  */
 
@@ -163,7 +186,6 @@ static void *
 pool_poolherder(void *priv)
 {
 	unsigned nwq;
-	VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
 	struct pool *pp;
 	uint64_t u;
 
@@ -175,7 +197,9 @@ pool_poolherder(void *priv)
 		if (nwq < cache_param->wthread_pools) {
 			pp = pool_mkpool(nwq);
 			if (pp != NULL) {
+				Lck_Lock(&pool_mtx);
 				VTAILQ_INSERT_TAIL(&pools, pp, list);
+				Lck_Unlock(&pool_mtx);
 				VSC_C_main->pools++;
 				nwq++;
 				continue;
@@ -183,7 +207,10 @@ pool_poolherder(void *priv)
 		}
 		/* XXX: remove pools */
 		if (0) {
+			Lck_Lock(&pool_mtx);
 			pp = VTAILQ_FIRST(&pools);
+			VTAILQ_REMOVE(&pools, pp, list);
+			Lck_Unlock(&pool_mtx);
 			AN(pp);
 			MPL_Destroy(&pp->mpl_sess);
 			MPL_Destroy(&pp->mpl_req);
@@ -191,8 +218,10 @@ pool_poolherder(void *priv)
 		}
 		(void)sleep(1);
 		u = 0;
+		Lck_Lock(&pool_mtx);
 		VTAILQ_FOREACH(pp, &pools, list)
 			u += pp->lqueue;
+		Lck_Unlock(&pool_mtx);
 		VSC_C_main->thread_queue_len = u;
 	}
 	NEEDLESS_RETURN(NULL);
@@ -207,4 +236,6 @@ Pool_Init(void)
 	Lck_New(&wstat_mtx, lck_wstat);
 	Lck_New(&pool_mtx, lck_wq);
 	AZ(pthread_create(&thr_pool_herder, NULL, pool_poolherder, NULL));
+	while (!VSC_C_main->pools)
+		(void)usleep(10000);
 }



From phk at FreeBSD.org  Sun May 31 06:33:42 2015
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 31 May 2015 08:33:42 +0200
Subject: [master] 8b122ca Add a needless return for gcc
Message-ID: <E1YywoY-0004uu-GV@project.varnish-software.com>


commit 8b122caa338680d535671585aa0c58f9809e69e6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun May 31 06:33:32 2015 +0000

    Add a needless return for gcc

diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index 8928d38..b6a0235 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -338,6 +338,7 @@ vbp_thread(struct worker *wrk, void *priv)
 			// XXX: ehh... ?
 		}
 	}
+	NEEDLESS_RETURN(NULL);
 }