From tfheen at varnish-software.com  Mon Dec  2 08:04:28 2013
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Mon, 02 Dec 2013 09:04:28 +0100
Subject: [3.0] 8433a20 Update params
Message-ID: <E1VnOUW-0006Tc-Pv@project.varnish-software.com>


commit 8433a2035707ed74c7d7bcd3066f929547f4da63
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 2 09:00:48 2013 +0100

    Update params

diff --git a/doc/sphinx/reference/params.rst b/doc/sphinx/reference/params.rst
index b5670fe..4fdd3e6 100644
--- a/doc/sphinx/reference/params.rst
+++ b/doc/sphinx/reference/params.rst
@@ -247,7 +247,7 @@ http_req_size
 	- Default: 32768
 
 	Maximum number of bytes of HTTP client request we will deal with.  This is a limit on all bytes up to the double blank line which ends the HTTP request.
-	The memory for the request is allocated from the client workspace (param: workspace_client) and this parameter limits how much of that the request is allowed to take up.
+	The memory for the request is allocated from the session workspace (param: sess_workspace) and this parameter limits how much of that the request is allowed to take up.
 
 http_resp_hdr_len
 	- Units: bytes
@@ -260,7 +260,7 @@ http_resp_size
 	- Default: 32768
 
 	Maximum number of bytes of HTTP backend resonse we will deal with.  This is a limit on all bytes up to the double blank line which ends the HTTP request.
-	The memory for the request is allocated from the thread pool workspace (param: thread_pool_workspace) and this parameter limits how much of that the request is allowed to take up.
+	The memory for the request is allocated from the worker workspace (param: sess_workspace) and this parameter limits how much of that the request is allowed to take up.
 
 idle_send_timeout
 	- Units: seconds
@@ -394,6 +394,14 @@ sess_timeout
 
 	Idle timeout for persistent sessions. If a HTTP request has not been received in this many seconds, the session is closed.
 
+sess_workspace
+	- Units: bytes
+	- Default: 65536
+	- Flags: delayed
+
+	Bytes of HTTP protocol workspace allocated for sessions. This space must be big enough for the entire HTTP protocol header and any edits done to it in the VCL code.
+	Minimum is 1024 bytes.
+
 session_linger
 	- Units: ms
 	- Default: 50



From tfheen at varnish-software.com  Mon Dec  2 08:04:28 2013
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Mon, 02 Dec 2013 09:04:28 +0100
Subject: [3.0] cafbd19 Changes for 3.0.5
Message-ID: <E1VnOUW-0006Ti-Ri@project.varnish-software.com>


commit cafbd197e64569158087e854e74621ce2169f984
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 2 09:01:15 2013 +0100

    Changes for 3.0.5

diff --git a/doc/changes.rst b/doc/changes.rst
index 99a9f66..e2398d1 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -1,4 +1,16 @@
 ================================
+Changes from 3.0.5 rc 1 to 3.0.5
+================================
+
+varnishd
+--------
+
+- Always check the local address of a socket.  This avoids a crash if
+  server.ip is accessed after a client has closed the connection. `Bug #1376`
+
+.. _bug #1376: http://varnish-cache.org/trac/ticket/1376
+
+================================
 Changes from 3.0.4 to 3.0.5 rc 1
 ================================
 



From tfheen at varnish-software.com  Mon Dec  2 08:04:28 2013
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Mon, 02 Dec 2013 09:04:28 +0100
Subject: [3.0] 1a89b1f Update version numbers for 3.0.5
Message-ID: <E1VnOUW-0006Tr-T9@project.varnish-software.com>


commit 1a89b1f75895bbf874e83cfc6f6123737a3fd76f
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 2 09:01:28 2013 +0100

    Update version numbers for 3.0.5

diff --git a/configure.ac b/configure.ac
index 5a454ef..83cbb47 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2011 Varnish Software AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [3.0.5-rc1], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [3.0.5], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/varnishapi.h)
 AM_CONFIG_HEADER(config.h)
 
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index aecf4f6..a8521ad 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -1,10 +1,9 @@
-%define v_rc rc1
 %define    _use_internal_dependency_generator 0
 %define __find_provides %{_builddir}/varnish-%{version}%{?v_rc:-%{?v_rc}}/redhat/find-provides
 Summary: High-performance HTTP accelerator
 Name: varnish
 Version: 3.0.5
-Release: 0.rc1%{?dist}
+Release: 1%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: http://www.varnish-cache.org/



From tfheen at varnish-software.com  Mon Dec  2 14:02:59 2013
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Mon, 02 Dec 2013 15:02:59 +0100
Subject: [master] c47b72a Make rpm build process somewhat more verbose
Message-ID: <E1VnU5T-0005oC-PE@project.varnish-software.com>


commit c47b72acdc21e4773d0adda9fdb06898b2489066
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 2 15:02:57 2013 +0100

    Make rpm build process somewhat more verbose

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 362edcd..26e7450 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -111,7 +111,7 @@ export CFLAGS="$CFLAGS -Wp,-D_FORTIFY_SOURCE=0"
 #sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=""|g;
 #	s|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool
 
-make %{?_smp_mflags}
+make %{?_smp_mflags} V=1
 
 head -6 etc/default.vcl > redhat/default.vcl
 
@@ -153,7 +153,7 @@ rm -rf doc/sphinx/\=build
 	%endif
 %endif
 
-make check LD_LIBRARY_PATH="../../lib/libvarnish/.libs:../../lib/libvarnishcompat/.libs:../../lib/libvarnishapi/.libs:../../lib/libvcc/.libs:../../lib/libvgz/.libs"
+make check LD_LIBRARY_PATH="../../lib/libvarnish/.libs:../../lib/libvarnishcompat/.libs:../../lib/libvarnishapi/.libs:../../lib/libvcc/.libs:../../lib/libvgz/.libs" TESTS_PARALLELISM=1 VERBOSE=1
 
 %install
 rm -rf %{buildroot}



From phk at FreeBSD.org  Tue Dec  3 09:25:53 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 10:25:53 +0100
Subject: [master] 942874e Always enable the sigsegv catcher in varnishtest
Message-ID: <E1VnmEr-0003rB-6l@project.varnish-software.com>


commit 942874e8136cea446bfa9a40f50c2dc56a0ae1dc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 3 09:25:39 2013 +0000

    Always enable the sigsegv catcher in varnishtest

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index a958fb0..3348b8e 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -382,6 +382,7 @@ varnish_launch(struct varnish *v)
 	VSB_printf(vsb, " -l 2m,1m,-");
 	VSB_printf(vsb, " -p auto_restart=off");
 	VSB_printf(vsb, " -p syslog_cli_traffic=off");
+	VSB_printf(vsb, " -p sigsegv_handler=on");
 	VSB_printf(vsb, " -a '%s'", "127.0.0.1:0");
 	VSB_printf(vsb, " -M '%s %s'", abuf, pbuf);
 	VSB_printf(vsb, " -P %s/varnishd.pid", v->workdir);




From tfheen at err.no  Mon Dec  2 14:07:13 2013
From: tfheen at err.no (Tollef Fog Heen)
Date: Mon, 02 Dec 2013 15:07:13 +0100
Subject: [3.0] e973102 Fix up dh_gencontrol call
Message-ID: <E1VnU9Z-0006b2-52@project.varnish-software.com>


commit e97310204b02370e1952bf76e7f710962b720a32
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 2 15:07:11 2013 +0100

    Fix up dh_gencontrol call

diff --git a/rules b/rules
index bb7fba4..115dddc 100755
--- a/rules
+++ b/rules
@@ -69,7 +69,7 @@ override_dh_gencontrol:
 	if [ -n "$$DEBIAN_OVERRIDE_BINARY_VERSION" ]; then \
 		dh_gencontrol -- -Tdebian/substvars -v$$DEBIAN_OVERRIDE_BINARY_VERSION; \
 	else \
-		dh_gencontrol -Tdebian/substvars; \
+		dh_gencontrol -- -Tdebian/substvars; \
 	fi
 
 # Override to add several init scripts




From phk at FreeBSD.org  Tue Dec  3 10:18:39 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 11:18:39 +0100
Subject: [master] 141001e Drop -Winline.  Omnios fails it,
 but it is not critical.
Message-ID: <E1Vnn3v-0002Ht-1L@project.varnish-software.com>


commit 141001effbf6d330bcec7cc477540063c04d4c14
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 3 10:18:16 2013 +0000

    Drop -Winline.  Omnios fails it, but it is not critical.

diff --git a/configure.ac b/configure.ac
index 51baa8d..cd346b3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -466,7 +466,7 @@ AX_CHECK_COMPILE_FLAG([-Werror=unused-result],
       OCFLAGS="${OCFLAGS} -Wno-unused-result"])])
 
 # This corresponds to FreeBSD's WARNS level 6
-DEVELOPER_CFLAGS="-fstack-protector -Werror -Wall -Wno-format-y2k -W -Wno-unused-parameter -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith -Wreturn-type -Wcast-qual -Wwrite-strings -Wswitch -Wshadow -Wunused-parameter -Wcast-align -Wchar-subscripts -Winline -Wnested-externs -Wno-pointer-sign -Wextra -Wno-missing-field-initializers -Wno-sign-compare"
+DEVELOPER_CFLAGS="-fstack-protector -Werror -Wall -Wno-format-y2k -W -Wno-unused-parameter -Wstrict-prototypes -Wmissing-prototypes -Wpointer-arith -Wreturn-type -Wcast-qual -Wwrite-strings -Wswitch -Wshadow -Wunused-parameter -Wcast-align -Wchar-subscripts -Wnested-externs -Wno-pointer-sign -Wextra -Wno-missing-field-initializers -Wno-sign-compare"
 
 # These are not compliable yet
 DEVELOPER_GCC_CFLAGS="-Wold-style-definition -Wredundant-decls "



From apj at mutt.dk  Tue Dec  3 14:55:19 2013
From: apj at mutt.dk (Andreas Plesner Jacobsen)
Date: Tue, 03 Dec 2013 15:55:19 +0100
Subject: [master] ea43572 Fix query string logging
Message-ID: <E1VnrNf-0004VP-JK@project.varnish-software.com>


commit ea435729e8b988c91da744bf6d1ba61877d4620b
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Dec 3 15:53:46 2013 +0100

    Fix query string logging

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 43412fc..452cc70 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -758,8 +758,8 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				p = memchr(b, '?', e - b);
 				if (p == NULL)
 					p = e;
-				frag_line(b, e, &CTX.frag[F_U]);
-				frag_line(b, e, &CTX.frag[F_q]);
+				frag_line(b, p, &CTX.frag[F_U]);
+				frag_line(p, e, &CTX.frag[F_q]);
 				break;
 			case SLT_ReqProtocol:
 				frag_line(b, e, &CTX.frag[F_H]);



From tfheen at varnish-software.com  Wed Dec  4 13:49:35 2013
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Wed, 04 Dec 2013 14:49:35 +0100
Subject: [master] b34a29d Fix typo, thanks to fgs
Message-ID: <E1VoCpb-0007FL-S3@project.varnish-software.com>


commit b34a29dc7ea70785075fc65da9f6529f8ed8702d
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Dec 4 14:49:27 2013 +0100

    Fix typo, thanks to fgs

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 79449fc..7e1a5e2 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -130,7 +130,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_ABANDON) {
 		if (bo->req != NULL)
 			vbf_release_req(bo);
-		(void)VFP_Error(bo, "Abandonned in vcl_backend_fetch");
+		(void)VFP_Error(bo, "Abandoned in vcl_backend_fetch");
 		return (F_STP_DONE);
 	}
 	assert (wrk->handling == VCL_RET_FETCH);



From phk at FreeBSD.org  Tue Dec  3 11:41:08 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 12:41:08 +0100
Subject: [master] cf2c6b9 Fix a NULL pointer deref in LRU.
Message-ID: <E1VnoLk-0001fv-9X@project.varnish-software.com>


commit cf2c6b97c0a0018ff4eea475d919fabaacbf856c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 3 11:40:40 2013 +0000

    Fix a NULL pointer deref in LRU.
    
    Spotted by: c0004x.vtc on slow machines.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 88d49fb..75364fd 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -331,6 +331,8 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 
 	VSLb(bo->vsl, SLT_ExpKill, "LRU x=%u",
 	    oc_getxid(bo->stats, oc) & VSL_IDENTMASK);
+	AN(bo->stats);
+	AN(oc);
 	(void)HSH_DerefObjCore(bo->stats, &oc);
 	return (1);
 }
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 8a738cc..e8b4914 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -384,8 +384,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		bo->exp.keep = 0.0;
 		obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
 	}
-	bo->stats = NULL;
 	if (obj == NULL) {
+		bo->stats = NULL;
 		(void)VFP_Error(bo, "Could not get storage");
 		VDI_CloseFd(&bo->vbc);
 		return (F_STP_DONE);
@@ -459,6 +459,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		break;
 	}
 
+	bo->stats = NULL;
+
 	bo->t_body = VTIM_mono();
 
 	if (bo->vbc != NULL) {
diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index 9021e6b..e170d89 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -178,6 +178,7 @@ VFP_GetStorage(struct busyobj *bo, ssize_t sz)
 	if (st != NULL && st->len < st->space)
 		return (st);
 
+	AN(bo->stats);
 	l = fetchfrag;
 	if (l == 0)
 		l = sz;
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index dd9d7d3..c54b982 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -202,6 +202,7 @@ stv_alloc_obj(struct busyobj *bo, size_t size)
 	 * Always use the stevedore which allocated the object in order to
 	 * keep an object inside the same stevedore.
 	 */
+	AN(bo->stats);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	obj = bo->fetch_obj;
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);



From phk at FreeBSD.org  Tue Dec  3 12:33:48 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 13:33:48 +0100
Subject: [master] bf8b317 Forgot the conditional case in previous fix.
Message-ID: <E1VnpAi-0006jR-Mh@project.varnish-software.com>


commit bf8b317fb81a6253d39f3196b6b087ee75e70bee
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 3 12:33:30 2013 +0000

    Forgot the conditional case in previous fix.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index e8b4914..ae72c62 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -562,7 +562,6 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		VDI_CloseFd(&bo->vbc);
 		return (F_STP_DONE);
 	}
-	bo->stats = NULL;
 
 	AZ(bo->fetch_obj);
 	bo->fetch_obj = obj;
@@ -621,6 +620,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		}
 	} while (ois == OIS_DATA || ois == OIS_STREAM);
 	ObjIterEnd(&oi);
+	bo->stats = NULL;
 	assert(al == bo->ims_obj->len);
 	assert(obj->len == al);
 	if (bo->state != BOS_FAILED)



From phk at FreeBSD.org  Tue Dec  3 16:38:41 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 17:38:41 +0100
Subject: [master] 146d103 Convert remaining timeouts to double format.
Message-ID: <E1Vnszh-000875-DS@project.varnish-software.com>


commit 146d103553841380b3beb059673ae98122f4bbff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 2 08:17:11 2013 +0000

    Convert remaining timeouts to double format.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index ba203c9..8f39e7c 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -173,13 +173,13 @@ vca_tcp_opt_init(void)
 #endif
 #ifdef HAVE_TCP_KEEP
 		} else if (!strcmp(to->strname, "TCP_KEEPIDLE")) {
-			x = cache_param->tcp_keepalive_time;
+			x = (int)(cache_param->tcp_keepalive_time);
 			NEW_VAL(to, x);
 		} else if (!strcmp(to->strname, "TCP_KEEPCNT")) {
-			x = cache_param->tcp_keepalive_probes;
+			x = (int)(cache_param->tcp_keepalive_probes);
 			NEW_VAL(to, x);
 		} else if (!strcmp(to->strname, "TCP_KEEPINTVL")) {
-			x = cache_param->tcp_keepalive_intvl;
+			x = (int)(cache_param->tcp_keepalive_intvl);
 			NEW_VAL(to, x);
 #endif
 		}
@@ -386,6 +386,7 @@ vca_acct(void *arg)
 {
 	struct listen_sock *ls;
 	double t0, now;
+	unsigned u;
 	int i;
 
 	THR_SetName("cache-acceptor");
@@ -398,12 +399,12 @@ vca_acct(void *arg)
 			continue;
 		AZ(listen(ls->sock, cache_param->listen_depth));
 #ifdef HAVE_TCP_KEEP
-		vca_tcp_keep_probe(ls->sock,
-		    TCP_KEEPIDLE, &cache_param->tcp_keepalive_time);
+		vca_tcp_keep_probe(ls->sock, TCP_KEEPIDLE, &u);
+		cache_param->tcp_keepalive_time = u;
 		vca_tcp_keep_probe(ls->sock,
 		    TCP_KEEPCNT, &cache_param->tcp_keepalive_probes);
-		vca_tcp_keep_probe(ls->sock,
-		    TCP_KEEPINTVL, &cache_param->tcp_keepalive_intvl);
+		vca_tcp_keep_probe(ls->sock, TCP_KEEPINTVL, &u);
+		cache_param->tcp_keepalive_intvl = u;
 #endif
 		vca_tcp_opt_set(ls->sock, 1);
 		if (cache_param->accept_filter) {
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 75364fd..49890bd 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -187,7 +187,7 @@ EXP_Touch(struct objcore *oc, double now)
 	if (oc->busyobj != NULL)
 		return;
 
-	if (now - oc->last_lru < cache_param->lru_timeout)
+	if (now - oc->last_lru < cache_param->lru_interval)
 		return;
 
 	lru = oc_getlru(oc);
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index b14c8af..a049196 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -110,7 +110,7 @@ PipeRequest(struct req *req, struct busyobj *bo)
 	while (fds[0].fd > -1 || fds[1].fd > -1) {
 		fds[0].revents = 0;
 		fds[1].revents = 0;
-		i = poll(fds, 2, cache_param->pipe_timeout * 1000);
+		i = poll(fds, 2, (int)(cache_param->pipe_timeout * 1e3));
 		if (i < 1)
 			break;
 		if (fds[0].revents && rdf(vc->fd, req->sp->fd)) {
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index b4129eb..8b075d2 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -107,13 +107,13 @@ struct params {
 	double			timeout_linger;
 	double			timeout_idle;
 	double			timeout_req;
-	unsigned		pipe_timeout;
-	unsigned		send_timeout;
-	unsigned		idle_send_timeout;
+	double			pipe_timeout;
+	double			send_timeout;
+	double			idle_send_timeout;
 #ifdef HAVE_TCP_KEEP
-	unsigned		tcp_keepalive_time;
+	double			tcp_keepalive_time;
 	unsigned		tcp_keepalive_probes;
-	unsigned		tcp_keepalive_intvl;
+	double			tcp_keepalive_intvl;
 #endif
 
 	/* Management hints */
@@ -133,12 +133,12 @@ struct params {
 	unsigned		listen_depth;
 
 	/* CLI related */
-	unsigned		cli_timeout;
+	double			cli_timeout;
 	unsigned		cli_limit;
 	unsigned		ping_interval;
 
 	/* LRU list ordering interval */
-	unsigned		lru_timeout;
+	double			lru_interval;
 
 	/* Maximum restarts allowed */
 	unsigned		max_restarts;
diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index d98b573..3263537 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -60,7 +60,6 @@ tweak_t tweak_listen_address;
 tweak_t tweak_poolparam;
 tweak_t tweak_string;
 tweak_t tweak_timeout;
-tweak_t tweak_timeout_double;
 tweak_t tweak_uint;
 tweak_t tweak_user;
 tweak_t tweak_waiter;
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index 4bb6592..9d46259 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -59,20 +59,20 @@ struct parspec mgt_parspec[] = {
 		"The unprivileged group to run as.",
 		MUST_RESTART,
 		"" },
-	{ "default_ttl", tweak_timeout_double, &mgt_param.default_ttl,
+	{ "default_ttl", tweak_timeout, &mgt_param.default_ttl,
 		"0", NULL,
 		"The TTL assigned to objects if neither the backend nor "
 		"the VCL code assigns one.",
 		OBJ_STICKY,
 		"20", "seconds" },
-	{ "default_grace", tweak_timeout_double, &mgt_param.default_grace,
+	{ "default_grace", tweak_timeout, &mgt_param.default_grace,
 		"0", NULL,
 		"Default grace period.  We will deliver an object "
 		"this long after it has expired, provided another thread "
 		"is attempting to get a new copy.",
 		OBJ_STICKY,
 		"10", "seconds" },
-	{ "default_keep", tweak_timeout_double, &mgt_param.default_keep,
+	{ "default_keep", tweak_timeout, &mgt_param.default_keep,
 		"0", NULL,
 		"Default keep period.  We will keep a useless object "
 		"around this long, making it available for conditional "
@@ -172,14 +172,14 @@ struct parspec mgt_parspec[] = {
 		"Maximum is 65535 bytes.",
 		0,
 		"255", "bytes" },
-	{ "timeout_idle", tweak_timeout_double, &mgt_param.timeout_idle,
+	{ "timeout_idle", tweak_timeout, &mgt_param.timeout_idle,
 		"0", NULL,
 		"Idle timeout for client connections.\n"
 		"A connection is considered idle, until we receive"
 		" a non-white-space character on it.",
 		0,
 		"5", "seconds" },
-	{ "timeout_req", tweak_timeout_double, &mgt_param.timeout_req,
+	{ "timeout_req", tweak_timeout, &mgt_param.timeout_req,
 		"0", NULL,
 		"Max time to receive clients request header, measured"
 		" from first non-white-space character to double CRNL.",
@@ -310,7 +310,7 @@ struct parspec mgt_parspec[] = {
 		"it possible to attach a debugger to the child.",
 		MUST_RESTART,
 		"3", "seconds" },
-	{ "lru_interval", tweak_timeout, &mgt_param.lru_timeout,
+	{ "lru_interval", tweak_timeout, &mgt_param.lru_interval,
 		"0", NULL,
 		"Grace period before object moves on LRU list.\n"
 		"Objects are only moved to the front of the LRU "
@@ -344,7 +344,7 @@ struct parspec mgt_parspec[] = {
 		"Maximum depth of esi:include processing.",
 		0,
 		"5", "levels" },
-	{ "connect_timeout", tweak_timeout_double, &mgt_param.connect_timeout,
+	{ "connect_timeout", tweak_timeout, &mgt_param.connect_timeout,
 		"0", NULL,
 		"Default connection timeout for backend connections. "
 		"We only try to connect to the backend for this many "
@@ -353,7 +353,7 @@ struct parspec mgt_parspec[] = {
 		"backend request.",
 		0,
 		"3.5", "s" },
-	{ "first_byte_timeout", tweak_timeout_double,
+	{ "first_byte_timeout", tweak_timeout,
 		&mgt_param.first_byte_timeout,
 		"0", NULL,
 		"Default timeout for receiving first byte from backend. "
@@ -364,7 +364,7 @@ struct parspec mgt_parspec[] = {
 		"backend request. This parameter does not apply to pipe.",
 		0,
 		"60", "s" },
-	{ "between_bytes_timeout", tweak_timeout_double,
+	{ "between_bytes_timeout", tweak_timeout,
 		&mgt_param.between_bytes_timeout,
 		"0", NULL,
 		"Default timeout between bytes when receiving data from "
@@ -375,7 +375,7 @@ struct parspec mgt_parspec[] = {
 		"and backend request. This parameter does not apply to pipe.",
 		0,
 		"60", "s" },
-	{ "acceptor_sleep_max", tweak_timeout_double,
+	{ "acceptor_sleep_max", tweak_timeout,
 		&mgt_param.acceptor_sleep_max,
 		"0", "10",
 		"If we run out of resources, such as file descriptors or "
@@ -384,7 +384,7 @@ struct parspec mgt_parspec[] = {
 		"attempts to accept new connections.",
 		EXPERIMENTAL,
 		"0.050", "s" },
-	{ "acceptor_sleep_incr", tweak_timeout_double,
+	{ "acceptor_sleep_incr", tweak_timeout,
 		&mgt_param.acceptor_sleep_incr,
 		"0", "1",
 		"If we run out of resources, such as file descriptors or "
@@ -424,7 +424,7 @@ struct parspec mgt_parspec[] = {
 		"it.",
 		0,
 		"100000", "sessions" },
-	{ "timeout_linger", tweak_timeout_double, &mgt_param.timeout_linger,
+	{ "timeout_linger", tweak_timeout, &mgt_param.timeout_linger,
 		"0", NULL,
 		"How long time the workerthread lingers on an idle session "
 		"before handing it over to the waiter.\n"
@@ -451,7 +451,7 @@ struct parspec mgt_parspec[] = {
 		"Log all CLI traffic to syslog(LOG_INFO).",
 		0,
 		"on", "bool" },
-	{ "ban_lurker_sleep", tweak_timeout_double,
+	{ "ban_lurker_sleep", tweak_timeout,
 		&mgt_param.ban_lurker_sleep,
 		"0", NULL,
 		"How long time does the ban lurker thread sleeps between "
@@ -502,14 +502,14 @@ struct parspec mgt_parspec[] = {
 		" just a waste of memory.",
 		EXPERIMENTAL,
 		"32k", "bytes" },
-	{ "shortlived", tweak_timeout_double,
+	{ "shortlived", tweak_timeout,
 		&mgt_param.shortlived,
 		"0", NULL,
 		"Objects created with TTL shorter than this are always "
 		"put in transient storage.",
 		0,
 		"10.0", "s" },
-	{ "critbit_cooloff", tweak_timeout_double,
+	{ "critbit_cooloff", tweak_timeout,
 		&mgt_param.critbit_cooloff,
 		"60", "254",
 		"How long time the critbit hasher keeps deleted objheads "
diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c
index 55520f4..4e2645f 100644
--- a/bin/varnishd/mgt/mgt_param_tweak.c
+++ b/bin/varnishd/mgt/mgt_param_tweak.c
@@ -104,25 +104,7 @@ tweak_generic_double(struct vsb *vsb, volatile double *dest,
 /*--------------------------------------------------------------------*/
 
 int
-tweak_timeout(struct vsb *vsb, const struct parspec *par, const char *arg)
-{
-	int i;
-	double d;
-	volatile unsigned *dest;
-
-	dest = par->priv;
-	d = *dest;
-	i = tweak_generic_double(vsb, &d, arg, par->min, par->max, "%.0f");
-	if (!i) {
-		*dest = (unsigned)ceil(d);
-	}
-	return (i);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-tweak_timeout_double(struct vsb *vsb, const struct parspec *par,
+tweak_timeout(struct vsb *vsb, const struct parspec *par,
     const char *arg)
 {
 	volatile double *dest;
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 91c0982..c3fd066 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -126,7 +126,7 @@ struct parspec WRK_parspec[] = {
 		DELAYED_EFFECT,
 		"100", "threads" },
 	{ "thread_pool_timeout",
-		tweak_timeout_double, &mgt_param.wthread_timeout,
+		tweak_timeout, &mgt_param.wthread_timeout,
 		"10", NULL,
 		"Thread idle threshold.\n"
 		"\n"
@@ -137,7 +137,7 @@ struct parspec WRK_parspec[] = {
 		EXPERIMENTAL | DELAYED_EFFECT,
 		"300", "seconds" },
 	{ "thread_pool_destroy_delay",
-		tweak_timeout_double, &mgt_param.wthread_destroy_delay,
+		tweak_timeout, &mgt_param.wthread_destroy_delay,
 		"0.01", NULL,
 		"Wait this long after destroying a thread.\n"
 		"\n"
@@ -147,7 +147,7 @@ struct parspec WRK_parspec[] = {
 		EXPERIMENTAL | DELAYED_EFFECT,
 		"1", "seconds" },
 	{ "thread_pool_add_delay",
-		tweak_timeout_double, &mgt_param.wthread_add_delay,
+		tweak_timeout, &mgt_param.wthread_add_delay,
 		"0", NULL,
 		"Wait at least this long after creating a thread.\n"
 		"\n"
@@ -160,7 +160,7 @@ struct parspec WRK_parspec[] = {
 		EXPERIMENTAL,
 		"0", "seconds" },
 	{ "thread_pool_fail_delay",
-		tweak_timeout_double, &mgt_param.wthread_fail_delay,
+		tweak_timeout, &mgt_param.wthread_fail_delay,
 		"10e-3", NULL,
 		"Wait at least this long after a failed thread creation "
 		"before trying to create another thread.\n"



From phk at FreeBSD.org  Tue Dec  3 16:38:41 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 17:38:41 +0100
Subject: [master] 6a92b33 Add a session workspace for the TCP addresses etc.
Message-ID: <E1Vnszh-00087C-Fc@project.varnish-software.com>


commit 6a92b339ffc5b084b1de410135dd9a054c97affe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 2 11:32:47 2013 +0000

    Add a session workspace for the TCP addresses etc.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8983e18..d6e0526 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -766,6 +766,8 @@ struct sess {
 
 	/* Session related fields ------------------------------------*/
 
+	struct ws		ws[1];
+
 	struct suckaddr		*remote_addr;
 	struct suckaddr		*local_addr;
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index ad0f537..fb488af 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -51,8 +51,6 @@
 #include "vtcp.h"
 #include "vtim.h"
 
-static unsigned ses_size;
-
 /*--------------------------------------------------------------------*/
 
 struct sesspool {
@@ -94,34 +92,28 @@ SES_Charge(struct worker *wrk, struct req *req)
  *
  * Layout is:
  *	struct sess
- *	struct vsa (local_addr)
- *	struct vsa (remote_addr)
+ *	workspace
  */
 
 static struct sess *
 ses_new(struct sesspool *pp)
 {
 	struct sess *sp;
-	char *s;
 	unsigned sz;
+	char *p, *e;
 
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	sp = MPL_Get(pp->mpl_sess, &sz);
 	sp->magic = SESS_MAGIC;
 	sp->sesspool = pp;
 
-	s = (char *)sp;
-	s += sizeof *sp;
-
-	memset(s, 0, vsa_suckaddr_len);
-	sp->local_addr = (void*)s;
-	s += vsa_suckaddr_len;
-
-	memset(s, 0, vsa_suckaddr_len);
-	sp->remote_addr = (void*)s;
-	s += vsa_suckaddr_len;
-
-	assert((char *)sp + sz == s);
+	e = (char*)sp + sz;
+	p = (char*)(sp + 1);
+	p = (void*)PRNDUP(p);
+	assert(p < e);
+	WS_Init(sp->ws, "sess", p, e - p);
+	sp->local_addr = (void*)WS_Alloc(sp->ws, vsa_suckaddr_len);
+	sp->remote_addr = (void*)WS_Alloc(sp->ws, vsa_suckaddr_len);
 
 	sp->t_open = NAN;
 	sp->t_idle = NAN;
@@ -454,8 +446,8 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	pp->mpl_req = MPL_New(nb, &cache_param->req_pool,
 	    &cache_param->workspace_client);
 	bprintf(nb, "sess%u", pool_no);
-	ses_size = sizeof (struct sess) + vsa_suckaddr_len * 2L;
-	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool, &ses_size);
+	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool,
+	    &cache_param->workspace_session);
 	return (pp);
 }
 
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index 8b075d2..58c7e9b 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -89,8 +89,9 @@ struct params {
 	unsigned		wthread_queue_limit;
 
 	/* Memory allocation hints */
-	unsigned		workspace_client;
 	unsigned		workspace_backend;
+	unsigned		workspace_client;
+	unsigned		workspace_session;
 	unsigned		workspace_thread;
 
 	unsigned		vsl_buffer;
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index 9d46259..d242efa 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -81,6 +81,13 @@ struct parspec mgt_parspec[] = {
 		"cache at the end of ttl+grace+keep.",
 		OBJ_STICKY,
 		"0", "seconds" },
+	{ "workspace_session",
+		tweak_bytes_u, &mgt_param.workspace_session,
+		"256", NULL,
+		"Bytes of workspace for session and TCP connection addresses."
+		"  If larger than 4k, use a multiple of 4k for VM efficiency.",
+		DELAYED_EFFECT,
+		"512", "bytes" },
 	{ "workspace_client",
 		tweak_bytes_u, &mgt_param.workspace_client,
 		"3072", NULL,



From phk at FreeBSD.org  Tue Dec  3 16:38:41 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 17:38:41 +0100
Subject: [master] 58aafc2 Allocate client IP/port string from sess->ws
Message-ID: <E1Vnszh-00087L-Hf@project.varnish-software.com>


commit 58aafc2a7ee750bada7a64f43378d5c6b7916e86
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 2 12:08:50 2013 +0000

    Allocate client IP/port string from sess->ws

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index d6e0526..9a6dc78 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -772,8 +772,8 @@ struct sess {
 	struct suckaddr		*local_addr;
 
 	/* formatted ascii client address */
-	char			addr[ADDR_BUFSIZE];
-	char			port[PORT_BUFSIZE];
+	char			*client_addr_str;
+	char			*client_port_str;
 
 	struct acct		acct_ses;
 
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 3258033..6f26703 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -348,7 +348,8 @@ pan_sess(const struct sess *sp)
 	VSB_printf(pan_vsp, "  sp = %p {\n", sp);
 	VSB_printf(pan_vsp, "    fd = %d, vxid = %u,\n",
 	    sp->fd, sp->vxid & VSL_IDENTMASK);
-	VSB_printf(pan_vsp, "    client = %s %s,\n", sp->addr, sp->port);
+	VSB_printf(pan_vsp, "    client = %s %s,\n", sp->client_addr_str,
+		sp->client_port_str);
 	switch (sp->sess_step) {
 #define SESS_STEP(l, u) case S_STP_##u: stp = "S_STP_" #u; break;
 #include "tbl/steps.h"
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 5502a4b..af8531f 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -674,7 +674,8 @@ cnt_recv(struct worker *wrk, struct req *req)
 	AZ(req->obj);
 	AZ(req->objcore);
 
-	VSLb(req->vsl, SLT_ReqStart, "%s %s", req->sp->addr, req->sp->port);
+	VSLb(req->vsl, SLT_ReqStart, "%s %s",
+	    req->sp->client_addr_str, req->sp->client_port_str);
 
 	if (req->err_code) {
 		req->req_step = R_STP_ERROR;
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index fb488af..23a1feb 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -192,12 +192,14 @@ ses_vsl_socket(struct sess *sp, const char *lsockname)
 	AN(VSA_Build(sp->local_addr, &ss, sl));
 	assert(VSA_Sane(sp->local_addr));
 
-	VTCP_name(sp->remote_addr,
-	    sp->addr, sizeof sp->addr, sp->port, sizeof sp->port);
+	VTCP_name(sp->remote_addr, laddr, sizeof laddr, lport, sizeof lport);
+	sp->client_addr_str = WS_Copy(sp->ws, laddr, -1);
+	sp->client_port_str = WS_Copy(sp->ws, lport, -1);
 	VTCP_name(sp->local_addr, laddr, sizeof laddr, lport, sizeof lport);
 	VSL(SLT_Begin, sp->vxid, "sess");
 	VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d",
-	    sp->addr, sp->port, lsockname, laddr, lport, sp->t_open, sp->fd);
+	    sp->client_addr_str, sp->client_port_str, lsockname, laddr, lport,
+	    sp->t_open, sp->fd);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 5e67e76..a769a06 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -199,7 +199,7 @@ VRT_r_client_identity(const struct vrt_ctx *ctx)
 	if (ctx->req->client_identity != NULL)
 		return (ctx->req->client_identity);
 	else
-		return (ctx->req->sp->addr);
+		return (ctx->req->sp->client_addr_str);
 }
 
 void



From phk at FreeBSD.org  Tue Dec  3 16:38:41 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 17:38:41 +0100
Subject: [master] 58687c6 Reduce the reporting in SLT_SessClose to be boolean
 bits.
Message-ID: <E1Vnszh-00087U-JM@project.varnish-software.com>


commit 58687c69d6b893986ebe50551c11292b9699cba9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 2 13:20:21 2013 +0000

    Reduce the reporting in SLT_SessClose to be boolean bits.
    
    This saves approx 50 bytes of storage for the session structure
    and puts us comfortably below 256 bytes, at least i the IPv4 case.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 9a6dc78..bf7fa0f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -246,6 +246,12 @@ struct acct {
 #undef ACCT
 };
 
+struct acct_bit {
+#define ACCT(foo)	unsigned int	foo : 1;
+#include "tbl/acct_fields.h"
+#undef ACCT
+};
+
 /*--------------------------------------------------------------------*/
 
 #define L0(t, n)
@@ -775,7 +781,7 @@ struct sess {
 	char			*client_addr_str;
 	char			*client_port_str;
 
-	struct acct		acct_ses;
+	struct acct_bit		acct_bit;
 
 	/* Timestamps, all on TIM_real() timescale */
 	double			t_open;		/* fd accepted */
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 23a1feb..ae41fd1 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -81,7 +81,7 @@ SES_Charge(struct worker *wrk, struct req *req)
 
 #define ACCT(foo)				\
 	wrk->stats.s_##foo += a->foo;		\
-	sp->acct_ses.foo += a->foo;		\
+	if (a->foo) sp->acct_bit.foo =1;	\
 	a->foo = 0;
 #include "tbl/acct_fields.h"
 #undef ACCT
@@ -312,7 +312,7 @@ SES_Close(struct sess *sp, enum sess_close reason)
 void
 SES_Delete(struct sess *sp, enum sess_close reason, double now)
 {
-	struct acct *b;
+	struct acct_bit *b;
 	struct sesspool *pp;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -328,8 +328,8 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
 		now = VTIM_real();
 	assert(!isnan(sp->t_open));
 
-	b = &sp->acct_ses;
-	VSL(SLT_SessClose, sp->vxid, "%s %.3f %ju %ju %ju %ju %ju %ju",
+	b = &sp->acct_bit;
+	VSL(SLT_SessClose, sp->vxid, "%s %.3f %u %u %u %u %u %u",
 	    sess_close_2str(sp->reason, 0), now - sp->t_open, b->req,
 	    b->pipe, b->pass, b->fetch, b->hdrbytes, b->bodybytes);
 	VSL(SLT_End, sp->vxid, "%s", "");
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index d242efa..baae9ee 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -87,7 +87,7 @@ struct parspec mgt_parspec[] = {
 		"Bytes of workspace for session and TCP connection addresses."
 		"  If larger than 4k, use a multiple of 4k for VM efficiency.",
 		DELAYED_EFFECT,
-		"512", "bytes" },
+		"256", "bytes" },
 	{ "workspace_client",
 		tweak_bytes_u, &mgt_param.workspace_client,
 		"3072", NULL,
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 85b0343..b39028a 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -103,12 +103,12 @@ SLTM(SessClose, 0, "Client connection closed",
 	"The format is::\n\n"
 	"\t%s %f %u %u %u %u %u %u\n"
 	"\t|  |  |  |  |  |  |  |\n"
-	"\t|  |  |  |  |  |  |  +- Body bytes sent on session\n"
-	"\t|  |  |  |  |  |  +---- Header bytes sent on session\n"
-	"\t|  |  |  |  |  +------- Backend fetches by session\n"
-	"\t|  |  |  |  +---------- Requests handled with pass\n"
-	"\t|  |  |  +------------- If 'pipe' were used on session\n"
-	"\t|  |  +---------------- How many requests on session\n"
+	"\t|  |  |  |  |  |  |  +- Bool: Body bytes were sent\n"
+	"\t|  |  |  |  |  |  +---- Bool: Resp.Header bytes sent\n"
+	"\t|  |  |  |  |  +------- Bool: Backend fetches initiated\n"
+	"\t|  |  |  |  +---------- Bool: 'pass' used\n"
+	"\t|  |  |  +------------- Bool: 'pipe' used\n"
+	"\t|  |  +---------------- Bool: good requests completed\n"
 	"\t|  +------------------- How long the session was open\n"
 	"\t+---------------------- Why the connection closed\n"
 	"\n"



From phk at FreeBSD.org  Tue Dec  3 16:38:41 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 17:38:41 +0100
Subject: [master] ddb9ff2 Up the sess workspace to 384,
 256 isn't enough for IPv6 (yet?)
Message-ID: <E1Vnszh-00087d-L5@project.varnish-software.com>


commit ddb9ff2b4a154ed0e292da21deaaaa7162b03db3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 2 23:01:45 2013 +0000

    Up the sess workspace to 384, 256 isn't enough for IPv6 (yet?)

diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index baae9ee..1859bec 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -87,7 +87,7 @@ struct parspec mgt_parspec[] = {
 		"Bytes of workspace for session and TCP connection addresses."
 		"  If larger than 4k, use a multiple of 4k for VM efficiency.",
 		DELAYED_EFFECT,
-		"256", "bytes" },
+		"384", "bytes" },
 	{ "workspace_client",
 		tweak_bytes_u, &mgt_param.workspace_client,
 		"3072", NULL,



From phk at FreeBSD.org  Tue Dec  3 16:38:41 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 17:38:41 +0100
Subject: [master] 30c6037 Only show the accept_filter parameter if the kernel
 can have accept-filters
Message-ID: <E1Vnszh-00087m-Mh@project.varnish-software.com>


commit 30c603790d43ea8d7e5e2f686e6982ddebc2b052
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 3 09:38:07 2013 +0000

    Only show the accept_filter parameter if the kernel can have
    accept-filters

diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index 1859bec..09dbe80 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -270,11 +270,13 @@ struct parspec mgt_parspec[] = {
 		"fragmentation.",
 		EXPERIMENTAL,
 		"256m", "bytes" },
+#ifdef HAVE_ACCEPT_FILTERS
 	{ "accept_filter", tweak_bool, &mgt_param.accept_filter,
 		NULL, NULL,
-		"Enable kernel accept-filters, if supported by the kernel.",
+		"Enable kernel accept-filters, (if available in the kernel).",
 		MUST_RESTART,
 		"on", "bool" },
+#endif
 	{ "listen_address", tweak_listen_address, NULL,
 		NULL, NULL,
 		"Whitespace separated list of network endpoints where "



From phk at FreeBSD.org  Tue Dec  3 16:38:41 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 17:38:41 +0100
Subject: [master] a328ddd Shave 8 bytes of struct ws on 64 bit archs.
Message-ID: <E1Vnszh-00087v-OP@project.varnish-software.com>


commit a328ddd5e0ee28959615c051981ff94057071fcf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 3 10:05:30 2013 +0000

    Shave 8 bytes of struct ws on 64 bit archs.
    
    With 8 byte pointers, having a char[X] for X <= 8 is cheaper than
    than a char *.
    
    Don't waste 32bits on the overflow flag.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bf7fa0f..eb7136b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -170,8 +170,7 @@ struct lock { void *priv; };	// Opaque
 struct ws {
 	unsigned		magic;
 #define WS_MAGIC		0x35fac554
-	unsigned		overflow;	/* workspace overflowed */
-	const char		*id;		/* identity */
+	char			id[4];		/* identity */
 	char			*s;		/* (S)tart of buffer */
 	char			*f;		/* (F)ree/front pointer */
 	char			*r;		/* (R)eserved length */
@@ -1207,6 +1206,7 @@ void WS_Reset(struct ws *ws, char *p);
 char *WS_Alloc(struct ws *ws, unsigned bytes);
 void *WS_Copy(struct ws *ws, const void *str, int len);
 char *WS_Snapshot(struct ws *ws);
+int WS_Overflowed(const struct ws *ws);
 
 /* rfc2616.c */
 void RFC2616_Ttl(struct busyobj *);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index ae72c62..79449fc 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -433,7 +433,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	assert(bo->refcount >= 1);
 
-	AZ(bo->ws_o->overflow);
+	AZ(WS_Overflowed(bo->ws_o));
 	if (bo->do_stream)
 		HSH_Unbusy(&wrk->stats, obj->objcore);
 
@@ -587,7 +587,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	http_CopyHome(obj->http);
 
 
-	AZ(bo->ws_o->overflow);
+	AZ(WS_Overflowed(bo->ws_o));
 	VBO_setstate(bo, BOS_FETCHING);
 	HSH_Unbusy(&wrk->stats, obj->objcore);
 
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 6f26703..97775ae 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -108,9 +108,10 @@ static void
 pan_ws(const struct ws *ws, int indent)
 {
 
-	VSB_printf(pan_vsp, "%*sws = %p { %s\n", indent, "",
-	    ws, ws->overflow ? "overflow" : "");
-	VSB_printf(pan_vsp, "%*sid = \"%s\",\n", indent + 2, "", ws->id);
+	VSB_printf(pan_vsp, "%*sws = %p {", indent, "", ws);
+	if (WS_Overflowed(ws))
+		VSB_printf(pan_vsp, " OVERFLOW");
+	VSB_printf(pan_vsp, "\n%*sid = \"%s\",\n", indent + 2, "", ws->id);
 	VSB_printf(pan_vsp, "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s);
 	if (ws->f > ws->s)
 		VSB_printf(pan_vsp, ",+%ld", (long) (ws->f - ws->s));
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index ae41fd1..46cb31f 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -111,7 +111,7 @@ ses_new(struct sesspool *pp)
 	p = (char*)(sp + 1);
 	p = (void*)PRNDUP(p);
 	assert(p < e);
-	WS_Init(sp->ws, "sess", p, e - p);
+	WS_Init(sp->ws, "ses", p, e - p);
 	sp->local_addr = (void*)WS_Alloc(sp->ws, vsa_suckaddr_len);
 	sp->remote_addr = (void*)WS_Alloc(sp->ws, vsa_suckaddr_len);
 
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 31738e3..17245b1 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -56,6 +56,11 @@ WS_Assert(const struct ws *ws)
 	}
 }
 
+/*
+ * NB: The id must be max 3 char and lower-case.
+ * (upper-case the first char to indicate overflow)
+ */
+
 void
 WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
 {
@@ -70,10 +75,21 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
 	ws->e = ws->s + len;
 	assert(PAOK(len));
 	ws->f = ws->s;
-	ws->id = id;
+	assert(id[0] & 0x40);
+	assert(strlen(id) < sizeof ws->id);
+	strcpy(ws->id, id);
 	WS_Assert(ws);
 }
 
+
+static void
+WS_MarkOverflow(struct ws *ws)
+{
+	WS_Assert(ws);
+
+	ws->id[0] &= ~0x40;		// Cheasy toupper()
+}
+
 /*
  * Reset a WS to start or a given pointer, likely from WS_Snapshot
  */
@@ -105,7 +121,7 @@ WS_Alloc(struct ws *ws, unsigned bytes)
 
 	assert(ws->r == NULL);
 	if (ws->f + bytes > ws->e) {
-		ws->overflow++;
+		WS_MarkOverflow(ws);
 		WS_Assert(ws);
 		return(NULL);
 	}
@@ -131,7 +147,7 @@ WS_Copy(struct ws *ws, const void *str, int len)
 
 	bytes = PRNDUP((unsigned)len);
 	if (ws->f + bytes > ws->e) {
-		ws->overflow++;
+		WS_MarkOverflow(ws);
 		WS_Assert(ws);
 		return(NULL);
 	}
@@ -201,3 +217,12 @@ WS_ReleaseP(struct ws *ws, char *ptr)
 	ws->r = NULL;
 	WS_Assert(ws);
 }
+int
+WS_Overflowed(const struct ws *ws)
+{
+	WS_Assert(ws);
+
+	if (ws->id[0] & 0x40)
+		return (0);
+	return (1);
+}



From phk at FreeBSD.org  Tue Dec  3 16:38:41 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 17:38:41 +0100
Subject: [master] d9a7f69 Pack struct sess better.
Message-ID: <E1Vnszh-000885-QG@project.varnish-software.com>


commit d9a7f69aa41707100db5237c2cf9574945782340
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 3 10:26:16 2013 +0000

    Pack struct sess better.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index eb7136b..b99e460 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -762,6 +762,8 @@ struct sess {
 	enum sess_close		reason;
 	uint32_t		vxid;
 
+	struct acct_bit		acct_bit;
+
 	/* Cross references ------------------------------------------*/
 
 	struct sesspool		*sesspool;
@@ -780,7 +782,6 @@ struct sess {
 	char			*client_addr_str;
 	char			*client_port_str;
 
-	struct acct_bit		acct_bit;
 
 	/* Timestamps, all on TIM_real() timescale */
 	double			t_open;		/* fd accepted */



From phk at FreeBSD.org  Tue Dec  3 16:38:41 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Dec 2013 17:38:41 +0100
Subject: [master] d75ca3b Sequeze struct sess storage harder. When we add the
 extra {local|remote}.ip for PROXY support,
 we do not want to waste 16 bytes extra on pointers,
 so put all the {client|server|local|remote}.ip into a single array -- which is
 unfortunately complicated by the opaqueness of struct vsa.
Message-ID: <E1Vnszh-00088E-S1@project.varnish-software.com>


commit d75ca3b4fcb28412ec50a8c6b1e054a9e4c43e8e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 3 10:45:54 2013 +0000

    Sequeze struct sess storage harder.  When we add the extra
    {local|remote}.ip for PROXY support, we do not want to waste 16 bytes
    extra on pointers, so put all the {client|server|local|remote}.ip into
    a single array -- which is unfortunately complicated by the opaqueness
    of struct vsa.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b99e460..5cbc27b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -775,8 +775,15 @@ struct sess {
 
 	struct ws		ws[1];
 
-	struct suckaddr		*remote_addr;
-	struct suckaddr		*local_addr;
+	/*
+	 * This gets quite involved, but we don't want to waste space
+	 * on up to 4 pointers of 8 bytes in struct sess.
+	 */
+	char			*addrs;
+#define sess_remote_addr(sp) \
+	((struct suckaddr *)(void*)((sp)->addrs))
+#define sess_local_addr(sp) \
+	((struct suckaddr *)(void*)((sp)->addrs + vsa_suckaddr_len))
 
 	/* formatted ascii client address */
 	char			*client_addr_str;
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 8f39e7c..2f5237c 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -352,7 +352,7 @@ VCA_SetupSess(struct worker *wrk, struct sess *sp)
 	wa->acceptsock = -1;
 	retval = wa->acceptlsock->name;
 	assert(wa->acceptaddrlen <= vsa_suckaddr_len);
-	AN(VSA_Build(sp->remote_addr, &wa->acceptaddr, wa->acceptaddrlen));
+	AN(VSA_Build(sess_remote_addr(sp), &wa->acceptaddr, wa->acceptaddrlen));
 	vca_pace_good();
 	wrk->stats.sess_conn++;
 	WS_Release(wrk->aws, 0);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 46cb31f..7b38cef 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -112,8 +112,7 @@ ses_new(struct sesspool *pp)
 	p = (void*)PRNDUP(p);
 	assert(p < e);
 	WS_Init(sp->ws, "ses", p, e - p);
-	sp->local_addr = (void*)WS_Alloc(sp->ws, vsa_suckaddr_len);
-	sp->remote_addr = (void*)WS_Alloc(sp->ws, vsa_suckaddr_len);
+	sp->addrs = (void*)WS_Alloc(sp->ws, vsa_suckaddr_len * 2);
 
 	sp->t_open = NAN;
 	sp->t_idle = NAN;
@@ -186,16 +185,18 @@ ses_vsl_socket(struct sess *sp, const char *lsockname)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AN(lsockname);
 
-	AN(sp->local_addr);
+	AN(sp->addrs);
 	sl = sizeof ss;
 	AZ(getsockname(sp->fd, (void*)&ss, &sl));
-	AN(VSA_Build(sp->local_addr, &ss, sl));
-	assert(VSA_Sane(sp->local_addr));
+	AN(VSA_Build(sess_local_addr(sp), &ss, sl));
+	assert(VSA_Sane(sess_local_addr(sp)));
 
-	VTCP_name(sp->remote_addr, laddr, sizeof laddr, lport, sizeof lport);
+	VTCP_name(sess_remote_addr(sp), laddr, sizeof laddr,
+	    lport, sizeof lport);
 	sp->client_addr_str = WS_Copy(sp->ws, laddr, -1);
 	sp->client_port_str = WS_Copy(sp->ws, lport, -1);
-	VTCP_name(sp->local_addr, laddr, sizeof laddr, lport, sizeof lport);
+	VTCP_name(sess_local_addr(sp), laddr, sizeof laddr,
+	    lport, sizeof lport);
 	VSL(SLT_Begin, sp->vxid, "sess");
 	VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d",
 	    sp->client_addr_str, sp->client_port_str, lsockname, laddr, lport,
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index a769a06..aec97dd 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -40,6 +40,7 @@
 #include "cache_backend.h"
 #include "vrt.h"
 #include "vrt_obj.h"
+#include "vsa.h"
 
 static char vrt_hostname[255] = "";
 
@@ -501,7 +502,8 @@ VRT_r_client_ip(const struct vrt_ctx *ctx)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	return (ctx->req->sp->remote_addr);
+	CHECK_OBJ_NOTNULL(ctx->req->sp, SESS_MAGIC);
+	return (sess_remote_addr(ctx->req->sp));
 }
 
 VCL_IP
@@ -510,8 +512,8 @@ VRT_r_server_ip(const struct vrt_ctx *ctx)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	AN(ctx->req->sp->local_addr);
-	return (ctx->req->sp->local_addr);
+	CHECK_OBJ_NOTNULL(ctx->req->sp, SESS_MAGIC);
+	return (sess_local_addr(ctx->req->sp));
 }
 
 const char*



From phk at FreeBSD.org  Thu Dec  5 10:05:01 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 05 Dec 2013 11:05:01 +0100
Subject: [master] 2d22965 Fix a variable leaf uninitialized by previous commit.
Message-ID: <E1VoVnp-00026P-8K@project.varnish-software.com>


commit 2d229657c6f5b7e070a75483b5c90c208f60410a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 5 10:04:17 2013 +0000

    Fix a variable leaf uninitialized by previous commit.
    
    Spotted by:	Scoof (first) and Coverity (second)

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 2f5237c..6a601a8 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -41,6 +41,7 @@
 
 #include "config.h"
 
+#include <math.h>
 #include <stdlib.h>
 #include <sys/types.h>
 #include <netinet/in.h>
@@ -399,10 +400,14 @@ vca_acct(void *arg)
 			continue;
 		AZ(listen(ls->sock, cache_param->listen_depth));
 #ifdef HAVE_TCP_KEEP
+		u = (unsigned)round(cache_param->tcp_keepalive_time);
 		vca_tcp_keep_probe(ls->sock, TCP_KEEPIDLE, &u);
 		cache_param->tcp_keepalive_time = u;
+
 		vca_tcp_keep_probe(ls->sock,
 		    TCP_KEEPCNT, &cache_param->tcp_keepalive_probes);
+
+		u = (unsigned)round(cache_param->tcp_keepalive_intvl);
 		vca_tcp_keep_probe(ls->sock, TCP_KEEPINTVL, &u);
 		cache_param->tcp_keepalive_intvl = u;
 #endif



From phk at FreeBSD.org  Sat Dec  7 09:40:30 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 07 Dec 2013 10:40:30 +0100
Subject: [master] abb29f4 This is a megacommit which introduces VFP's:
 Fetch-Processors.
Message-ID: <E1VpENC-0000KI-Sz@project.varnish-software.com>


commit abb29f413f2e72647cf3f58e0153ed9c7e72587a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Dec 7 09:40:07 2013 +0000

    This is a megacommit which introduces VFP's: Fetch-Processors.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5cbc27b..08f81b2 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -123,7 +123,6 @@ struct req;
 struct sess;
 struct sesspool;
 struct vbc;
-struct vef_priv;
 struct vrt_backend;
 struct vsb;
 struct waitinglist;
@@ -265,20 +264,18 @@ struct dstat {
 
 /* Fetch processors --------------------------------------------------*/
 
-typedef void vfp_begin_f(struct busyobj *bo, size_t );
-typedef int vfp_bytes_f(struct busyobj *bo, struct http_conn *, ssize_t);
-typedef int vfp_end_f(struct busyobj *bo);
-
-struct vfp {
-	vfp_begin_f	*begin;
-	vfp_bytes_f	*bytes;
-	vfp_end_f	*end;
+enum vfp_status {
+	VFP_ERROR = -1,
+	VFP_OK = 0,
+	VFP_END = 1,
 };
+typedef enum vfp_status vfp_pull_f(struct busyobj *bo, void *p, ssize_t *len, intptr_t *priv);
 
-extern const struct vfp vfp_gunzip;
-extern const struct vfp vfp_gzip;
-extern const struct vfp vfp_testgzip;
-extern const struct vfp vfp_esi;
+extern vfp_pull_f vfp_gunzip_pull;
+extern vfp_pull_f vfp_gzip_pull;
+extern vfp_pull_f vfp_testgunzip_pull;
+extern vfp_pull_f vfp_esi_pull;
+extern vfp_pull_f vfp_esi_gzip_pull;
 
 /* Deliver processors ------------------------------------------------*/
 
@@ -545,10 +542,12 @@ struct busyobj {
 	unsigned		is_gzip;
 	unsigned		is_gunzip;
 
-	const struct vfp	*vfp;
-	struct vep_state	*vep;
+#define N_VFPS			5
+	vfp_pull_f		*vfps[N_VFPS];
+	intptr_t		vfps_priv[N_VFPS];
+	int			vfp_nxt;
+
 	enum busyobj_state_e	state;
-	struct vgz		*vgz_rx;
 
 	struct ws		ws[1];
 	struct vbc		*vbc;
@@ -564,8 +563,6 @@ struct busyobj {
 
 	struct pool_task	fetch_task;
 
-	struct vef_priv		*vef_priv;
-
 	unsigned		should_close;
 	char			*h_content_length;
 
@@ -859,7 +856,7 @@ void VBO_waitstate(struct busyobj *bo, enum busyobj_state_e want);
 
 /* cache_http1_fetch.c [V1F] */
 int V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req);
-void V1F_fetch_body(struct busyobj *bo);
+ssize_t V1F_Setup_Fetch(struct busyobj *bo);
 
 /* cache_http1_fsm.c [HTTP1] */
 typedef int (req_body_iter_f)(struct req *, void *priv, void *ptr, size_t);
@@ -944,10 +941,14 @@ void VBF_Fetch(struct worker *wrk, struct req *req,
 
 /* cache_fetch_proc.c */
 struct storage *VFP_GetStorage(struct busyobj *, ssize_t sz);
-int VFP_Error2(struct busyobj *, const char *error, const char *more);
-int VFP_Error(struct busyobj *, const char *error);
+enum vfp_status VFP_Error(struct busyobj *, const char *fmt, ...)
+    __printflike(2, 3);
 void VFP_Init(void);
-extern const struct vfp VFP_nop;
+void VFP_Fetch_Body(struct busyobj *bo, ssize_t est);
+void VFP_Push(struct busyobj *, vfp_pull_f *func, intptr_t priv);
+enum vfp_status VFP_Suck(struct busyobj *, void *p, ssize_t *lp);
+extern char vfp_init[];
+extern char vfp_fini[];
 
 /* cache_gzip.c */
 struct vgz;
@@ -966,7 +967,6 @@ void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
 int VGZ_IbufEmpty(const struct vgz *vg);
 void VGZ_Obuf(struct vgz *, void *, ssize_t len);
 int VGZ_ObufFull(const struct vgz *vg);
-int VGZ_ObufStorage(struct busyobj *, struct vgz *vg);
 enum vgzret_e VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
 enum vgzret_e VGZ_Gunzip(struct vgz *, const void **, size_t *len);
 enum vgzret_e VGZ_Destroy(struct vgz **);
@@ -1141,6 +1141,7 @@ void VSM_Free(void *ptr);
 #ifdef VSL_ENDMARKER
 void VSL(enum VSL_tag_e tag, uint32_t vxid, const char *fmt, ...)
     __printflike(3, 4);
+void VSLbv(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, va_list va);
 void VSLb(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, ...)
     __printflike(3, 4);
 void VSLbt(struct vsl_log *, enum VSL_tag_e tag, txt t);
diff --git a/bin/varnishd/cache/cache_esi.h b/bin/varnishd/cache/cache_esi.h
index 9903bd8..52c9d0b 100644
--- a/bin/varnishd/cache/cache_esi.h
+++ b/bin/varnishd/cache/cache_esi.h
@@ -39,8 +39,10 @@
 #define	VEC_S8	(0x60 + 8)
 #define	VEC_INCL	'I'
 
-typedef ssize_t vep_callback_t(struct busyobj *, ssize_t l, enum vgz_flag flg);
+typedef ssize_t vep_callback_t(struct busyobj *, void *priv, ssize_t l,
+    enum vgz_flag flg);
 
-void VEP_Init(struct busyobj *, vep_callback_t *cb);
-void VEP_Parse(const struct busyobj *, const char *p, size_t l);
-struct vsb *VEP_Finish(struct busyobj *);
+struct vep_state *VEP_Init(struct busyobj *, vep_callback_t *cb, void *cb_priv);
+void VEP_Parse(struct vep_state *, const struct busyobj *, const char *p,
+    size_t l);
+struct vsb *VEP_Finish(struct vep_state *, const struct busyobj *);
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 6cf8dbd..3eb4c49 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -44,6 +44,8 @@ struct vef_priv {
 #define VEF_MAGIC		0xf104b51f
 	struct vgz		*vgz;
 
+	struct vep_state	*vep;
+
 	ssize_t			tot;
 	int			error;
 
@@ -51,149 +53,19 @@ struct vef_priv {
 	char			*ibuf_i;
 	char			*ibuf_o;
 	ssize_t			ibuf_sz;
-
-	char			*ibuf2;
-	ssize_t			ibuf2_sz;
 };
 
-/*---------------------------------------------------------------------
- * Read some bytes.
- *
- * If the DBG_ESI_CHOP is set, we read only a couple of bytes at
- * a time, in order to stress the parse/pending/callback code.
- */
-
-static ssize_t
-vef_read(struct http_conn *htc, void *buf, ssize_t buflen, ssize_t bytes)
-{
-	ssize_t d;
-
-	if (buflen < bytes)
-		bytes = buflen;
-	if (DO_DEBUG(DBG_ESI_CHOP)) {
-		d = (random() & 3) + 1;
-		if (d < bytes)
-			bytes = d;
-	}
-	return (htc->read(htc, buf, bytes));
-}
-
-/*---------------------------------------------------------------------
- * We receive a ungzip'ed object, and want to store it ungzip'ed.
- */
-
-static int
-vfp_esi_bytes_uu(struct busyobj *bo, const struct vef_priv *vef,
-    struct http_conn *htc, ssize_t bytes)
-{
-	ssize_t wl;
-	struct storage *st;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-
-	while (bytes > 0) {
-		st = VFP_GetStorage(bo, 0);
-		if (st == NULL)
-			return (-1);
-		wl = vef_read(htc,
-		    st->ptr + st->len, st->space - st->len, bytes);
-		if (wl <= 0)
-			return (wl);
-		VEP_Parse(bo, (const char *)st->ptr + st->len, wl);
-		VBO_extend(bo, wl);
-		bytes -= wl;
-	}
-	return (1);
-}
-
-/*---------------------------------------------------------------------
- * We receive a gzip'ed object, and want to store it ungzip'ed.
- */
-
-static int
-vfp_esi_bytes_gu(struct busyobj *bo, const struct vef_priv *vef,
-    struct http_conn *htc, ssize_t bytes)
-{
-	struct vgz *vg;
-	ssize_t wl;
-	enum vgzret_e vr;
-	size_t dl;
-	const void *dp;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-	vg = bo->vgz_rx;
-
-	while (bytes > 0) {
-		if (VGZ_IbufEmpty(vg) && bytes > 0) {
-			wl = vef_read(htc, vef->ibuf, vef->ibuf_sz, bytes);
-			if (wl <= 0)
-				return (wl);
-			VGZ_Ibuf(vg, vef->ibuf, wl);
-			bytes -= wl;
-		}
-		if (VGZ_ObufStorage(bo, vg))
-			return(-1);
-		vr = VGZ_Gunzip(vg, &dp, &dl);
-		if (vr < VGZ_OK)
-			return (-1);
-		if (dl > 0) {
-			VEP_Parse(bo, dp, dl);
-			VBO_extend(bo, dl);
-		}
-	}
-	return (1);
-}
-
-/*---------------------------------------------------------------------
- * We receive a [un]gzip'ed object, and want to store it gzip'ed.
- *
- * This is rather complicated, because the ESI parser does not
- * spit out all bytes we feed it right away:  Sometimes it needs
- * more input to make up its mind.
- *
- * The inject function feeds uncompressed bytes into the VEP, and
- * takes care to keep any bytes VEP didn't decide on intact until
- * later.
- *
- * The callback is called by VEP to dispose of bytes and report
- * where to find them again later.
- */
-
-static int
-vfp_vep_inject(const struct busyobj *bo, struct vef_priv *vef, ssize_t wl)
-{
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-
-	VEP_Parse(bo, vef->ibuf_i, wl);
-	vef->ibuf_i += wl;
-	assert(vef->ibuf_o >= vef->ibuf && vef->ibuf_o <= vef->ibuf_i);
-	if (vef->error) {
-		errno = vef->error;
-		return (-1);
-	}
-	wl = vef->ibuf_i - vef->ibuf_o;
-	if (wl > 0)
-		memmove(vef->ibuf, vef->ibuf_o, wl);
-	vef->ibuf_o = vef->ibuf;
-	vef->ibuf_i = vef->ibuf + wl;
-	return (0);
-}
-
 static ssize_t
-vfp_vep_callback(struct busyobj *bo, ssize_t l, enum vgz_flag flg)
+vfp_vep_callback(struct busyobj *bo, void *priv, ssize_t l, enum vgz_flag flg)
 {
 	struct vef_priv *vef;
 	size_t dl;
 	const void *dp;
+	struct storage *st;
 	int i;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vef = bo->vef_priv;
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+	CAST_OBJ_NOTNULL(vef, priv, VEF_MAGIC);
 	assert(l >= 0);
 
 	if (vef->error) {
@@ -211,177 +83,36 @@ vfp_vep_callback(struct busyobj *bo, ssize_t l, enum vgz_flag flg)
 
 	VGZ_Ibuf(vef->vgz, vef->ibuf_o, l);
 	do {
-		if (VGZ_ObufStorage(bo, vef->vgz)) {
+		st = VFP_GetStorage(bo, 0);
+		if (st == NULL) {
 			vef->error = ENOMEM;
 			vef->tot += l;
 			return (vef->tot);
 		}
+		VGZ_Obuf(vef->vgz, st->ptr + st->len, st->space - st->len);
 		i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
 		vef->tot += dl;
 		VBO_extend(bo, dl);
-	} while (!VGZ_IbufEmpty(vef->vgz) ||
-	    (flg != VGZ_NORMAL && VGZ_ObufFull(vef->vgz)));
-	assert(VGZ_IbufEmpty(vef->vgz));
+	} while (i != VGZ_ERROR &&
+	    (!VGZ_IbufEmpty(vef->vgz) || VGZ_ObufFull(vef->vgz)));
+	assert(i == VGZ_ERROR || VGZ_IbufEmpty(vef->vgz));
 	vef->ibuf_o += l;
-	if (flg == VGZ_FINISH)
-		assert(i == 1);			/* XXX */
-	else
-		assert(i == 0);			/* XXX */
 	return (vef->tot);
 }
 
-/*---------------------------------------------------------------------
- * We receive a gunzip'ed object, and want to store it gzip'ed.
- */
-
-static int
-vfp_esi_bytes_ug(const struct busyobj *bo, struct vef_priv *vef,
-    struct http_conn *htc, ssize_t bytes)
-{
-	ssize_t wl;
-
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	while (bytes > 0) {
-		wl = vef->ibuf_sz - (vef->ibuf_i - vef->ibuf);
-		wl = vef_read(htc, vef->ibuf_i, wl, bytes);
-		if (wl <= 0)
-			return (wl);
-		bytes -= wl;
-		if (vfp_vep_inject(bo, vef, wl))
-			return (-1);
-	}
-	return (1);
-}
-
-/*---------------------------------------------------------------------
- * We receive a gzip'ed object, and want to store it gzip'ed.
- */
-
-static int
-vfp_esi_bytes_gg(const struct busyobj *bo, struct vef_priv *vef,
-    struct http_conn *htc, size_t bytes)
-{
-	ssize_t wl;
-	size_t dl;
-	const void *dp;
-	enum vgzret_e vr;
-
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	while (bytes > 0) {
-		wl = vef_read(htc, vef->ibuf2, vef->ibuf2_sz, bytes);
-		if (wl <= 0)
-			return (wl);
-		bytes -= wl;
-
-		VGZ_Ibuf(bo->vgz_rx, vef->ibuf2, wl);
-		do {
-			wl = vef->ibuf_sz - (vef->ibuf_i - vef->ibuf);
-			VGZ_Obuf(bo->vgz_rx, vef->ibuf_i, wl);
-			vr = VGZ_Gunzip(bo->vgz_rx, &dp, &dl);
-			if (vr < VGZ_OK)
-				return (-1);
-			if (dl > 0 && vfp_vep_inject(bo, vef, dl))
-				return (-1);
-		} while (!VGZ_IbufEmpty(bo->vgz_rx));
-	}
-	return (1);
-}
-
-/*---------------------------------------------------------------------*/
-
-static void __match_proto__(vfp_begin_f)
-vfp_esi_begin(struct busyobj *bo, size_t estimate)
-{
-	struct vef_priv *vef;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	(void)estimate;
-
-	ALLOC_OBJ(vef, VEF_MAGIC);
-	XXXAN(vef);
-	AZ(bo->vef_priv);
-	bo->vef_priv = vef;
-
-	AZ(bo->vgz_rx);
-	if (bo->is_gzip && bo->do_gunzip) {
-		bo->vgz_rx = VGZ_NewUngzip(bo->vsl, "U F E");
-		VEP_Init(bo, NULL);
-		vef->ibuf_sz = cache_param->gzip_buffer;
-	} else if (bo->is_gunzip && bo->do_gzip) {
-		vef->vgz = VGZ_NewGzip(bo->vsl, "G F E");
-		VEP_Init(bo, vfp_vep_callback);
-		vef->ibuf_sz = cache_param->gzip_buffer;
-	} else if (bo->is_gzip) {
-		bo->vgz_rx = VGZ_NewUngzip(bo->vsl, "U F E");
-		vef->vgz = VGZ_NewGzip(bo->vsl, "G F E");
-		VEP_Init(bo, vfp_vep_callback);
-		vef->ibuf_sz = cache_param->gzip_buffer;
-		vef->ibuf2_sz = cache_param->gzip_buffer;
-	} else {
-		VEP_Init(bo, NULL);
-	}
-	if (vef->ibuf_sz > 0) {
-		vef->ibuf = calloc(1L, vef->ibuf_sz);
-		XXXAN(vef->ibuf);
-		vef->ibuf_i = vef->ibuf;
-		vef->ibuf_o = vef->ibuf;
-	}
-	if (vef->ibuf2_sz > 0) {
-		vef->ibuf2 = calloc(1L, vef->ibuf2_sz);
-		XXXAN(vef->ibuf2);
-	}
-	AN(bo->vep);
-}
-
-static int __match_proto__(vfp_bytes_f)
-vfp_esi_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
-{
-	struct vef_priv *vef;
-	int i;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vef = bo->vef_priv;
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-
-	AN(bo->vep);
-	assert(&bo->htc == htc);
-	if (bo->is_gzip && bo->do_gunzip)
-		i = vfp_esi_bytes_gu(bo, vef, htc, bytes);
-	else if (bo->is_gunzip && bo->do_gzip)
-		i = vfp_esi_bytes_ug(bo, vef, htc, bytes);
-	else if (bo->is_gzip)
-		i = vfp_esi_bytes_gg(bo, vef, htc, bytes);
-	else
-		i = vfp_esi_bytes_uu(bo, vef, htc, bytes);
-	AN(bo->vep);
-	return (i);
-}
-
-static int __match_proto__(vfp_end_f)
-vfp_esi_end(struct busyobj *bo)
+static enum vfp_status
+vfp_esi_end(struct busyobj *bo, struct vef_priv *vef, enum vfp_status retval)
 {
 	struct vsb *vsb;
-	struct vef_priv *vef;
 	ssize_t l;
-	int retval = 0;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	AN(bo->vep);
-
-	if (bo->state == BOS_FAILED)
-		retval = -1;
-
-	if (bo->vgz_rx != NULL && VGZ_Destroy(&bo->vgz_rx) != VGZ_END)
-		retval = VFP_Error(bo, "Gunzip+ESI Failed at the very end");
+	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
-	vsb = VEP_Finish(bo);
+	vsb = VEP_Finish(vef->vep, bo);
 
 	if (vsb != NULL) {
-		if (!retval) {
+		if (retval == VFP_END) {
 			l = VSB_len(vsb);
 			assert(l > 0);
 			/* XXX: This is a huge waste of storage... */
@@ -398,9 +129,6 @@ vfp_esi_end(struct busyobj *bo)
 		VSB_delete(vsb);
 	}
 
-	vef = bo->vef_priv;
-	bo->vef_priv = NULL;
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 	if (vef->vgz != NULL) {
 		VGZ_UpdateObj(vef->vgz, bo->fetch_obj);
 		if (VGZ_Destroy(&vef->vgz) != VGZ_END)
@@ -408,15 +136,107 @@ vfp_esi_end(struct busyobj *bo)
 			    "ESI+Gzip Failed at the very end");
 	}
 	if (vef->ibuf != NULL)
-		free(vef->ibuf);
-	if (vef->ibuf2 != NULL)
-		free(vef->ibuf2);
 	FREE_OBJ(vef);
 	return (retval);
 }
 
-const struct vfp vfp_esi = {
-        .begin  =       vfp_esi_begin,
-        .bytes  =       vfp_esi_bytes,
-        .end    =       vfp_esi_end,
-};
+enum vfp_status __match_proto__(vfp_pull_f)
+vfp_esi_gzip_pull(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
+{
+	enum vfp_status vp;
+	ssize_t d, l;
+	struct vef_priv *vef;
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	if (p == vfp_init) {
+		ALLOC_OBJ(vef, VEF_MAGIC);
+		XXXAN(vef);
+		vef->vgz = VGZ_NewGzip(bo->vsl, "G F E");
+		vef->vep = VEP_Init(bo, vfp_vep_callback, vef);
+		vef->ibuf_sz = cache_param->gzip_buffer;
+		vef->ibuf = calloc(1L, vef->ibuf_sz);
+		XXXAN(vef->ibuf);
+		vef->ibuf_i = vef->ibuf;
+		vef->ibuf_o = vef->ibuf;
+		*priv = (uintptr_t)vef;
+		return (VFP_OK);
+	}
+	if (p == vfp_fini) {
+		if (*priv)
+			(void)vfp_esi_end(bo, (void*)*priv, VFP_ERROR);
+		*priv = 0;
+		return (VFP_ERROR);
+	}
+	AN(p);
+	AN(lp);
+	*lp = 0;
+	AN(priv);
+	CAST_OBJ_NOTNULL(vef, (void*)*priv, VEF_MAGIC);
+	l = vef->ibuf_sz - (vef->ibuf_i - vef->ibuf);
+	if (DO_DEBUG(DBG_ESI_CHOP)) {
+		d = (random() & 3) + 1;
+		if (d < l)
+			l = d;
+	}
+	vp = VFP_Suck(bo, vef->ibuf_i, &l);
+
+	if (l > 0) {
+		VEP_Parse(vef->vep, bo, vef->ibuf_i, l);
+		vef->ibuf_i += l;
+		assert(vef->ibuf_o >= vef->ibuf && vef->ibuf_o <= vef->ibuf_i);
+		if (vef->error) {
+			errno = vef->error;
+			return (VFP_ERROR);
+		}
+		l = vef->ibuf_i - vef->ibuf_o;
+		if (l > 0)
+			memmove(vef->ibuf, vef->ibuf_o, l);
+		vef->ibuf_o = vef->ibuf;
+		vef->ibuf_i = vef->ibuf + l;
+	}
+	if (vp == VFP_END) {
+		vp = vfp_esi_end(bo, vef, vp);
+		*priv = 0;
+	}
+	return (vp);
+}
+
+enum vfp_status __match_proto__(vfp_pull_f)
+vfp_esi_pull(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
+{
+	enum vfp_status vp;
+	ssize_t d;
+	struct vef_priv *vef;
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	if (p == vfp_init) {
+		ALLOC_OBJ(vef, VEF_MAGIC);
+		XXXAN(vef);
+		vef->vep = VEP_Init(bo, NULL, NULL);
+		*priv = (uintptr_t)vef;
+		return (VFP_OK);
+	}
+	if (p == vfp_fini) {
+		if (*priv)
+			(void)vfp_esi_end(bo, (void*)*priv, VFP_ERROR);
+		*priv = 0;
+		return (VFP_ERROR);
+	}
+	AN(p);
+	AN(lp);
+	AN(priv);
+	CAST_OBJ_NOTNULL(vef, (void*)*priv, VEF_MAGIC);
+	if (DO_DEBUG(DBG_ESI_CHOP)) {
+		d = (random() & 3) + 1;
+		if (d < *lp)
+			*lp = d;
+	}
+	vp = VFP_Suck(bo, p, lp);
+	if (vp != VFP_ERROR && *lp > 0)
+		VEP_Parse(vef->vep, bo, p, *lp);
+	if (vp == VFP_END) {
+		vp = vfp_esi_end(bo, vef, vp);
+		*priv = 0;
+	}
+	return (vp);
+}
diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index 1c900f6..66d3420 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -63,6 +63,7 @@ struct vep_state {
 	struct busyobj		*bo;
 	int			dogzip;
 	vep_callback_t		*cb;
+	void			*cb_priv;
 
 	/* Internal Counter for default call-back function */
 	ssize_t			cb_x;
@@ -329,7 +330,7 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 	 */
 
 	if (vep->last_mark != mark && (vep->o_wait > 0 || vep->startup)) {
-		lcb = vep->cb(vep->bo, 0,
+		lcb = vep->cb(vep->bo, vep->cb_priv, 0,
 		    mark == VERBATIM ? VGZ_RESET : VGZ_ALIGN);
 		if (lcb - vep->o_last > 0)
 			vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
@@ -339,7 +340,8 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 
 	/* Transfer pending bytes CRC into active mode CRC */
 	if (vep->o_pending) {
-		(void)vep->cb(vep->bo, vep->o_pending, VGZ_NORMAL);
+		(void)vep->cb(vep->bo, vep->cb_priv, vep->o_pending,
+		     VGZ_NORMAL);
 		if (vep->o_crc == 0) {
 			vep->crc = vep->crcp;
 			vep->o_crc = vep->o_pending;
@@ -363,7 +365,7 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 
 	vep->o_wait += l;
 	vep->last_mark = mark;
-	(void)vep->cb(vep->bo, l, VGZ_NORMAL);
+	(void)vep->cb(vep->bo, vep->cb_priv, l, VGZ_NORMAL);
 }
 
 static void
@@ -565,15 +567,14 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
  */
 
 void
-VEP_Parse(const struct busyobj *bo, const char *p, size_t l)
+VEP_Parse(struct vep_state *vep, const struct busyobj *bo, const char *p,
+    size_t l)
 {
-	struct vep_state *vep;
 	const char *e;
 	struct vep_match *vm;
 	int i;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vep = bo->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	assert(l > 0);
 
@@ -1013,29 +1014,27 @@ VEP_Parse(const struct busyobj *bo, const char *p, size_t l)
  */
 
 static ssize_t __match_proto__()
-vep_default_cb(struct busyobj *bo, ssize_t l, enum vgz_flag flg)
+vep_default_cb(struct busyobj *bo, void *priv, ssize_t l, enum vgz_flag flg)
 {
-	struct vep_state *vep;
+	ssize_t *s;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vep = bo->vep;
-	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
-	assert(vep->bo == bo);
+	AN(priv);
+	s = priv;
+	*s += l;
 	(void)flg;
-	vep->cb_x += l;
-	return (vep->cb_x);
+	return (*s);
 }
 
 /*---------------------------------------------------------------------
  */
 
-void
-VEP_Init(struct busyobj *bo, vep_callback_t *cb)
+struct vep_state *
+VEP_Init(struct busyobj *bo, vep_callback_t *cb, void *cb_priv)
 {
 	struct vep_state *vep;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	AZ(bo->vep);
 	vep = (void*)WS_Alloc(bo->ws, sizeof *vep);
 	AN(vep);
 
@@ -1044,15 +1043,16 @@ VEP_Init(struct busyobj *bo, vep_callback_t *cb)
 	vep->bo = bo;
 	vep->vsb = VSB_new_auto();
 	AN(vep->vsb);
-	bo->vep = vep;
 
 	if (cb != NULL) {
 		vep->dogzip = 1;
 		/* XXX */
 		VSB_printf(vep->vsb, "%c", VEC_GZ);
 		vep->cb = cb;
+		vep->cb_priv = cb_priv;
 	} else {
 		vep->cb = vep_default_cb;
+		vep->cb_priv = &vep->cb_x;
 	}
 
 	vep->state = VEP_START;
@@ -1069,31 +1069,29 @@ VEP_Init(struct busyobj *bo, vep_callback_t *cb)
 	vep->last_mark = SKIP;
 	vep_mark_common(vep, vep->ver_p, VERBATIM);
 	vep->startup = 0;
+	return (vep);
 }
 
 /*---------------------------------------------------------------------
  */
 
 struct vsb *
-VEP_Finish(struct busyobj *bo)
+VEP_Finish(struct vep_state *vep, const struct busyobj *bo)
 {
-	struct vep_state *vep;
 	ssize_t l, lcb;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vep = bo->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	assert(vep->bo == bo);
 
 	if (vep->o_pending)
 		vep_mark_common(vep, vep->ver_p, vep->last_mark);
 	if (vep->o_wait > 0) {
-		lcb = vep->cb(vep->bo, 0, VGZ_ALIGN);
+		lcb = vep->cb(vep->bo, vep->cb_priv, 0, VGZ_ALIGN);
 		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
 	}
-	(void)vep->cb(vep->bo, 0, VGZ_FINISH);
+	(void)vep->cb(vep->bo, vep->cb_priv, 0, VGZ_FINISH);
 
-	bo->vep = NULL;
 	AZ(VSB_finish(vep->vsb));
 	l = VSB_len(vep->vsb);
 	if (vep->esi_found && l > 0)
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 7e1a5e2..eb066eb 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -259,6 +259,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	struct vsb *vary = NULL;
 	int varyl = 0;
 	struct object *obj;
+	ssize_t est = -1;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -309,24 +310,27 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	/* But we can't do both at the same time */
 	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
 
-	/* ESI takes precedence and handles gzip/gunzip itself */
-	if (bo->do_esi) {
-		bo->vfp = &vfp_esi;
-		/*
-		 * The one case were we do not weaken Etag is where
-		 * incoming obj is not gzip'ed and we don't gzip either
-		 * If we ESI expand it on deliver, we weaken there.
-		 */
-		if (bo->is_gzip || bo->do_gzip | bo->do_gunzip)
-			RFC2616_Weaken_Etag(bo->beresp);
-	} else if (bo->do_gunzip) {
-		bo->vfp = &vfp_gunzip;
+	if (bo->vbc != NULL)
+		est = V1F_Setup_Fetch(bo);
+
+	if (bo->do_gunzip || (bo->is_gzip && bo->do_esi)) {
+		RFC2616_Weaken_Etag(bo->beresp);
+		VFP_Push(bo, vfp_gunzip_pull, 0);
+	}
+
+	if (bo->do_esi && bo->do_gzip) {
+		VFP_Push(bo, vfp_esi_gzip_pull, 0);
+		RFC2616_Weaken_Etag(bo->beresp);
+	} else if (bo->do_esi && bo->is_gzip && !bo->do_gunzip) {
+		VFP_Push(bo, vfp_esi_gzip_pull, 0);
 		RFC2616_Weaken_Etag(bo->beresp);
+	} else if (bo->do_esi) {
+		VFP_Push(bo, vfp_esi_pull, 0);
 	} else if (bo->do_gzip) {
-		bo->vfp = &vfp_gzip;
+		VFP_Push(bo, vfp_gzip_pull, 0);
 		RFC2616_Weaken_Etag(bo->beresp);
-	} else if (bo->is_gzip) {
-		bo->vfp = &vfp_testgzip;
+	} else if (bo->is_gzip && !bo->do_gunzip) {
+		VFP_Push(bo, vfp_testgunzip_pull, 0);
 	}
 
 	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
@@ -437,9 +441,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->do_stream)
 		HSH_Unbusy(&wrk->stats, obj->objcore);
 
-	if (bo->vfp == NULL)
-		bo->vfp = &VFP_nop;
-
 	assert(bo->state == BOS_REQ_DONE);
 	VBO_setstate(bo, BOS_FETCHING);
 
@@ -455,8 +456,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		if (bo->vbc == NULL)
 			(void)VFP_Error(bo, "Backend connection gone");
 		else
-			V1F_fetch_body(bo);
-		break;
+			VFP_Fetch_Body(bo, est);
 	}
 
 	bo->stats = NULL;
@@ -471,14 +471,12 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		AZ(bo->vbc);
 	}
 
-	bo->vfp = NULL;
+	http_Teardown(bo->bereq);
+	http_Teardown(bo->beresp);
 
 	VSLb(bo->vsl, SLT_Fetch_Body, "%u(%s)",
 	    bo->htc.body_status, body_status_2str(bo->htc.body_status));
 
-	http_Teardown(bo->bereq);
-	http_Teardown(bo->beresp);
-
 	if (bo->state == BOS_FAILED) {
 		wrk->stats.fetch_failed++;
 	} else {
diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index e170d89..d7c3610 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -43,6 +43,9 @@
 
 static unsigned fetchfrag;
 
+char vfp_init[] = "<init>";
+char vfp_fini[] = "<fini>";
+
 /*--------------------------------------------------------------------
  * We want to issue the first error we encounter on fetching and
  * supress the rest.  This function does that.
@@ -52,114 +55,24 @@ static unsigned fetchfrag;
  * For convenience, always return -1
  */
 
-int
-VFP_Error2(struct busyobj *bo, const char *error, const char *more)
+enum vfp_status
+VFP_Error(struct busyobj *bo, const char *fmt, ...)
 {
+	va_list ap;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	if (bo->state < BOS_FAILED) {
-		if (more == NULL)
-			VSLb(bo->vsl, SLT_FetchError, "%s", error);
-		else
-			VSLb(bo->vsl, SLT_FetchError, "%s: %s", error, more);
+		va_start(ap, fmt);
+		VSLbv(bo->vsl, SLT_FetchError, fmt, ap);
+		va_end(ap);
 		if (bo->fetch_objcore != NULL)
 			HSH_Fail(bo->fetch_objcore);
 		VBO_setstate(bo, BOS_FAILED);
 	}
-	return (-1);
-}
-
-int
-VFP_Error(struct busyobj *bo, const char *error)
-{
-	return(VFP_Error2(bo, error, NULL));
-}
-
-/*--------------------------------------------------------------------
- * VFP_NOP
- *
- * This fetch-processor does nothing but store the object.
- * It also documents the API
- */
-
-/*--------------------------------------------------------------------
- * VFP_BEGIN
- *
- * Called to set up stuff.
- *
- * 'estimate' is the estimate of the number of bytes we expect to receive,
- * as seen on the socket, or zero if unknown.
- */
-
-static void __match_proto__(vfp_begin_f)
-vfp_nop_begin(struct busyobj *bo, size_t estimate)
-{
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	if (estimate > 0)
-		(void)VFP_GetStorage(bo, estimate);
-}
-
-/*--------------------------------------------------------------------
- * VFP_BYTES
- *
- * Process (up to) 'bytes' from the socket.
- *
- * Return -1 on error, issue VFP_Error()
- *	will not be called again, once error happens.
- * Return 0 on EOF on socket even if bytes not reached.
- * Return 1 when 'bytes' have been processed.
- */
-
-static int __match_proto__(vfp_bytes_f)
-vfp_nop_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
-{
-	ssize_t l, wl;
-	struct storage *st;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-
-	while (bytes > 0) {
-		st = VFP_GetStorage(bo, 0);
-		if (st == NULL)
-			return(-1);
-		l = st->space - st->len;
-		if (l > bytes)
-			l = bytes;
-		wl = HTTP1_Read(htc, st->ptr + st->len, l);
-		if (wl <= 0)
-			return (wl);
-		VBO_extend(bo, wl);
-		bytes -= wl;
-	}
-	return (1);
+	return (VFP_ERROR);
 }
 
 /*--------------------------------------------------------------------
- * VFP_END
- *
- * Finish & cleanup
- *
- * Return -1 for error
- * Return 0 for OK
- */
-
-static int __match_proto__(vfp_end_f)
-vfp_nop_end(struct busyobj *bo)
-{
-
-	(void)bo;
-	return (0);
-}
-
-const struct vfp VFP_nop = {
-	.begin	=	vfp_nop_begin,
-	.bytes	=	vfp_nop_bytes,
-	.end	=	vfp_nop_end,
-};
-
-/*--------------------------------------------------------------------
  * Fetch Storage to put object into.
  *
  */
@@ -196,6 +109,160 @@ VFP_GetStorage(struct busyobj *bo, ssize_t sz)
 	return (st);
 }
 
+/**********************************************************************
+ */
+
+static enum vfp_status
+vfp_call(struct busyobj *bo, int nbr, void *p, ssize_t *lp)
+{
+	AN(bo->vfps[nbr]);
+	return (bo->vfps[nbr](bo, p, lp, &bo->vfps_priv[nbr]));
+}
+
+static void
+vfp_suck_fini(struct busyobj *bo)
+{
+	int i;
+
+	for (i = 0; i < bo->vfp_nxt; i++) {
+		if(bo->vfps[i] != NULL)
+			(void)vfp_call(bo, i, vfp_fini, NULL);
+	}
+}
+
+static enum vfp_status
+vfp_suck_init(struct busyobj *bo)
+{
+	enum vfp_status retval = VFP_ERROR;
+	int i;
+
+	for (i = 0; i < bo->vfp_nxt; i++) {
+		retval = vfp_call(bo, i, vfp_init, NULL);
+		if (retval != VFP_OK) {
+			vfp_suck_fini(bo);
+			break;
+		}
+	}
+	return (retval);
+}
+
+/**********************************************************************
+ * Suck data up from lower levels.
+ * Once a layer return non VFP_OK, clean it up and produce the same
+ * return value for any subsequent calls.
+ */
+
+enum vfp_status
+VFP_Suck(struct busyobj *bo, void *p, ssize_t *lp)
+{
+	enum vfp_status vp;
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	AN(p);
+	AN(lp);
+	assert(bo->vfp_nxt > 0);
+	bo->vfp_nxt--;
+	if (bo->vfps[bo->vfp_nxt] == NULL) {
+		*lp = 0;
+		vp = (enum vfp_status)bo->vfps_priv[bo->vfp_nxt];
+	} else {
+		vp = vfp_call(bo, bo->vfp_nxt, p, lp);
+		if (vp != VFP_OK) {
+			(void)vfp_call(bo, bo->vfp_nxt, vfp_fini, NULL);
+			bo->vfps[bo->vfp_nxt] = NULL;
+			bo->vfps_priv[bo->vfp_nxt] = vp;
+		}
+	}
+	bo->vfp_nxt++;
+	return (vp);
+}
+
+/*--------------------------------------------------------------------
+ */
+
+void
+VFP_Fetch_Body(struct busyobj *bo, ssize_t est)
+{
+	ssize_t l;
+	enum vfp_status vfps = VFP_ERROR;
+	struct storage *st = NULL;
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	AN(bo->vfp_nxt);
+
+	if (est < 0)
+		est = 0;
+
+	if (vfp_suck_init(bo) != VFP_OK) {
+		(void)VFP_Error(bo, "Fetch Pipeline failed to initialize");
+		bo->should_close = 1;
+		return;
+	}
+
+	do {
+		if (st == NULL) {
+			l = fetchfrag;
+			if (l == 0) {
+				l = est;
+				est = 0;
+			}
+			if (l == 0)
+				l = cache_param->fetch_chunksize;
+			st = STV_alloc(bo, l);
+			if (st == NULL) {
+				bo->should_close = 1;
+				/* XXX Close VFP stack */
+				(void)VFP_Error(bo, "Out of storage");
+				break;
+			}
+			AZ(st->len);
+			Lck_Lock(&bo->mtx);
+			VTAILQ_INSERT_TAIL(&bo->fetch_obj->store, st, list);
+			Lck_Unlock(&bo->mtx);
+		}
+		l = st->space - st->len;
+		vfps = VFP_Suck(bo, st->ptr + st->len, &l);
+		if (l > 0)
+			VBO_extend(bo, l);
+		if (st->len == st->space)
+			st = NULL;
+	} while (vfps == VFP_OK);
+
+	if (vfps == VFP_ERROR) {
+		(void)VFP_Error(bo, "Fetch Pipeline failed to process");
+		bo->should_close = 1;
+	}
+
+	vfp_suck_fini(bo);
+
+	/*
+	 * Trim or delete the last segment, if any
+	 */
+
+	st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
+	/* XXX: Temporary:  Only trim if we are not streaming */
+	if (st != NULL && !bo->do_stream) {
+		/* None of this this is safe under streaming */
+		if (st->len == 0) {
+			VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
+			STV_free(st);
+		} else if (st->len < st->space) {
+			STV_trim(st, st->len, 1);
+		}
+	}
+}
+
+void
+VFP_Push(struct busyobj *bo, vfp_pull_f *func, intptr_t priv)
+{
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	bo->vfps_priv[bo->vfp_nxt] = priv;
+	bo->vfps[bo->vfp_nxt] = func;
+	bo->vfp_nxt++;
+}
+
 /*--------------------------------------------------------------------
  * Debugging aids
  */
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 09f1594..bdbf229 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -51,9 +51,8 @@ struct vgz {
 	enum {VGZ_GZ,VGZ_UN}	dir;
 	struct vsl_log		*vsl;
 	const char		*id;
-	struct ws		*tmp;
-	char			*tmp_snapshot;
 	int			last_i;
+	enum vgz_flag		flag;
 
 	/* Wrw stuff */
 	char			*m_buf;
@@ -117,11 +116,6 @@ VGZ_NewGzip(struct vsl_log *vsl, const char *id)
 	 *
 	 * windowBits [8..15] (-> 1K..128K)
 	 * memLevel [1..9] (-> 1K->256K)
-	 *
-	 * XXX: They probably needs to be params...
-	 *
-	 * XXX: It may be more efficent to malloc them, rather than have
-	 * XXX: too many worker threads grow the stacks.
 	 */
 	i = deflateInit2(&vg->vz,
 	    cache_param->gzip_level,		/* Level */
@@ -195,24 +189,6 @@ VGZ_ObufFull(const struct vgz *vg)
 	return (vg->vz.avail_out == 0);
 }
 
-/*--------------------------------------------------------------------
- * Keep the outbuffer supplied with storage
- */
-
-int
-VGZ_ObufStorage(struct busyobj *bo, struct vgz *vg)
-{
-	struct storage *st;
-
-	st = VFP_GetStorage(bo, 0);
-	if (st == NULL)
-		return (-1);
-
-	VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
-
-	return (0);
-}
-
 /*--------------------------------------------------------------------*/
 
 enum vgzret_e
@@ -445,8 +421,6 @@ VGZ_Destroy(struct vgz **vgp)
 	    (intmax_t)vg->vz.start_bit,
 	    (intmax_t)vg->vz.last_bit,
 	    (intmax_t)vg->vz.stop_bit);
-	if (vg->tmp != NULL)
-		WS_Reset(vg->tmp, vg->tmp_snapshot);
 	if (vg->dir == VGZ_GZ)
 		i = deflateEnd(&vg->vz);
 	else
@@ -462,7 +436,8 @@ VGZ_Destroy(struct vgz **vgp)
 	else if (i == Z_BUF_ERROR)
 		vr = VGZ_STUCK;
 	else {
-		VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)", i, vg->vz.msg);
+		VSLb(vg->vsl, SLT_Gzip, "G(un)zip error: %d (%s)",
+		    i, vg->vz.msg);
 		vr = VGZ_ERROR;
 	}
 	FREE_OBJ(vg);
@@ -475,164 +450,139 @@ VGZ_Destroy(struct vgz **vgp)
  * A VFP for gunzip'ing an object as we receive it from the backend
  */
 
-static void __match_proto__(vfp_begin_f)
-vfp_gunzip_begin(struct busyobj *bo, size_t estimate)
-{
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	(void)estimate;
-	AZ(bo->vgz_rx);
-	bo->vgz_rx = VGZ_NewUngzip(bo->vsl, "U F -");
-	XXXAZ(vgz_getmbuf(bo->vgz_rx));
-}
-
-static int __match_proto__(vfp_bytes_f)
-vfp_gunzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
+enum vfp_status __match_proto__(vfp_pull_f)
+vfp_gunzip_pull(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 {
+        ssize_t l;
 	struct vgz *vg;
-	ssize_t l, wl;
-	int i = -100;
-	size_t dl;
+	enum vgzret_e vr = VGZ_ERROR;
 	const void *dp;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vg = bo->vgz_rx;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	AZ(vg->vz.avail_in);
-	while (bytes > 0 || vg->vz.avail_in > 0) {
-		if (vg->vz.avail_in == 0 && bytes > 0) {
-			l = vg->m_sz;
-			if (l > bytes)
-				l = bytes;
-			wl = htc->read(htc, vg->m_buf, l);
-			if (wl <= 0)
-				return (wl);
-			VGZ_Ibuf(vg, vg->m_buf, wl);
-			bytes -= wl;
-		}
-
-		if (VGZ_ObufStorage(bo, vg))
-			return(-1);
-		i = VGZ_Gunzip(vg, &dp, &dl);
-		if (i != VGZ_OK && i != VGZ_END)
-			return(VFP_Error(bo, "Gunzip data error"));
-		if (i == VGZ_END && !VGZ_IbufEmpty(vg))
-			return(VFP_Error(bo, "Junk after gzip data"));
-		VBO_extend(bo, dl);
+	size_t dl;
+	enum vfp_status vp = VFP_OK;
+
+        CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	if (p == vfp_init) {
+		vg = VGZ_NewUngzip(bo->vsl, "U F -");
+		XXXAZ(vgz_getmbuf(vg));
+		*priv = (uintptr_t)vg;
+		VGZ_Ibuf(vg, vg->m_buf, 0);
+		AZ(vg->m_len);
+		return (VFP_OK);
 	}
-	assert(i == Z_OK || i == Z_STREAM_END);
-	return (1);
-}
-
-static int __match_proto__(vfp_end_f)
-vfp_gunzip_end(struct busyobj *bo)
-{
-	struct vgz *vg;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vg = bo->vgz_rx;
-	bo->vgz_rx = NULL;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	if (bo->state == BOS_FAILED) {
-		(void)VGZ_Destroy(&vg);
-		return(0);
+	if (p == vfp_fini) {
+		if (*priv != 0) {
+			CAST_OBJ_NOTNULL(vg, (void*)(*priv), VGZ_MAGIC);
+			*priv = 0;
+			(void)VGZ_Destroy(&vg);
+		}
+		*priv = 0;
+		return (VFP_ERROR);
 	}
-	if (VGZ_Destroy(&vg) != VGZ_END)
+        AN(p);
+        AN(lp);
+        AN(priv);
+	CAST_OBJ_NOTNULL(vg, (void*)(*priv), VGZ_MAGIC);
+	l = *lp;
+	*lp = 0;
+	VGZ_Obuf(vg, p, l);
+	do {
+		if (VGZ_IbufEmpty(vg)) {
+			l = vg->m_sz;
+			vp = VFP_Suck(bo, vg->m_buf, &l);
+			if (vp == VFP_ERROR)
+				return (vp);
+			VGZ_Ibuf(vg, vg->m_buf, l);
+		}
+		if (!VGZ_IbufEmpty(vg) || vp == VFP_END) {
+			vr = VGZ_Gunzip(vg, &dp, &dl);
+			if (vr == VGZ_END && !VGZ_IbufEmpty(vg))
+				return(VFP_Error(bo, "Junk after gzip data"));
+			if (vr < VGZ_OK)
+				return (VFP_Error(bo,
+				    "Invalid Gzip data: %s", vg->vz.msg));
+			if (dl > 0) {
+				*lp = dl;
+				assert(dp == p);
+				return (VFP_OK);
+			}
+		}
+		AN(VGZ_IbufEmpty(vg));
+	} while (vp == VFP_OK);
+	if (vr != VGZ_END)
 		return(VFP_Error(bo, "Gunzip error at the very end"));
-	return (0);
+	return (vp);
 }
 
-const struct vfp vfp_gunzip = {
-        .begin  =       vfp_gunzip_begin,
-        .bytes  =       vfp_gunzip_bytes,
-        .end    =       vfp_gunzip_end,
-};
-
 /*--------------------------------------------------------------------
  * VFP_GZIP
  *
  * A VFP for gzip'ing an object as we receive it from the backend
  */
 
-static void __match_proto__(vfp_begin_f)
-vfp_gzip_begin(struct busyobj *bo, size_t estimate)
-{
-	(void)estimate;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	AZ(bo->vgz_rx);
-	bo->vgz_rx = VGZ_NewGzip(bo->vsl, "G F -");
-	XXXAZ(vgz_getmbuf(bo->vgz_rx));
-}
-
-static int __match_proto__(vfp_bytes_f)
-vfp_gzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
+enum vfp_status __match_proto__(vfp_pull_f)
+vfp_gzip_pull(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 {
+        ssize_t l;
 	struct vgz *vg;
-	ssize_t l, wl;
-	int i = -100;
-	size_t dl;
+	enum vgzret_e vr = VGZ_ERROR;
 	const void *dp;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vg = bo->vgz_rx;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	AZ(vg->vz.avail_in);
-	while (bytes > 0 || !VGZ_IbufEmpty(vg)) {
-		if (VGZ_IbufEmpty(vg) && bytes > 0) {
-			l = vg->m_sz;
-			if (l > bytes)
-				l = bytes;
-			wl = htc->read(htc, vg->m_buf, l);
-			if (wl <= 0)
-				return (wl);
-			VGZ_Ibuf(vg, vg->m_buf, wl);
-			bytes -= wl;
-		}
-		if (VGZ_ObufStorage(bo, vg))
-			return(-1);
-		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
-		assert(i == Z_OK);
-		VBO_extend(bo, dl);
-	}
-	return (1);
-}
-
-static int __match_proto__(vfp_end_f)
-vfp_gzip_end(struct busyobj *bo)
-{
-	struct vgz *vg;
 	size_t dl;
-	const void *dp;
-	int i;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vg = bo->vgz_rx;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	bo->vgz_rx = NULL;
-	if (bo->state == BOS_FAILED) {
-		(void)VGZ_Destroy(&vg);
-		return(0);
+	enum vfp_status vp = VFP_ERROR;
+
+        CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	if (p == vfp_init) {
+		vg = VGZ_NewGzip(bo->vsl, "G F -");
+		XXXAZ(vgz_getmbuf(vg));
+		*priv = (uintptr_t)vg;
+		VGZ_Ibuf(vg, vg->m_buf, 0);
+		AZ(vg->m_len);
+		vg->flag = VGZ_NORMAL;
+		return (VFP_OK);
+	}
+	if (p == vfp_fini) {
+		if (*priv != 0) {
+			CAST_OBJ_NOTNULL(vg, (void*)(*priv), VGZ_MAGIC);
+			*priv = 0;
+			(void)VGZ_Destroy(&vg);
+		}
+		return (VFP_ERROR);
 	}
+        AN(p);
+        AN(lp);
+        AN(priv);
+	CAST_OBJ_NOTNULL(vg, (void*)(*priv), VGZ_MAGIC);
+	l = *lp;
+	*lp = 0;
+	VGZ_Obuf(vg, p, l);
 	do {
-		VGZ_Ibuf(vg, "", 0);
-		if (VGZ_ObufStorage(bo, vg))
-			return(-1);
-		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-		VBO_extend(bo, dl);
-	} while (i != Z_STREAM_END);
+		if (VGZ_IbufEmpty(vg)) {
+			l = vg->m_sz;
+			vp = VFP_Suck(bo, vg->m_buf, &l);
+			if (vp == VFP_ERROR)
+				break;
+			if (vp == VFP_END)
+				vg->flag = VGZ_FINISH;
+			VGZ_Ibuf(vg, vg->m_buf, l);
+		}
+		if (!VGZ_IbufEmpty(vg) || vg->flag == VGZ_FINISH) {
+			vr = VGZ_Gzip(vg, &dp, &dl, vg->flag);
+			if (vr < VGZ_OK)
+				return (VFP_Error(bo, "Gzip failed"));
+			if (dl > 0) {
+				*lp = dl;
+				assert(dp == p);
+				return (VFP_OK);
+			}
+		}
+		AN(VGZ_IbufEmpty(vg));
+	} while (vg->flag != VGZ_FINISH);
+
+	if (vr != VGZ_END)
+		return (VFP_Error(bo, "Gzip failed"));
 	VGZ_UpdateObj(vg, bo->fetch_obj);
-	if (VGZ_Destroy(&vg) != VGZ_END)
-		return(VFP_Error(bo, "Gzip error at the very end"));
-	return (0);
+	return (VFP_END);
 }
 
-const struct vfp vfp_gzip = {
-        .begin  =       vfp_gzip_begin,
-        .bytes  =       vfp_gzip_bytes,
-        .end    =       vfp_gzip_end,
-};
-
 /*--------------------------------------------------------------------
  * VFP_TESTGZIP
  *
@@ -640,80 +590,54 @@ const struct vfp vfp_gzip = {
  * collecting the magic bits while we're at it.
  */
 
-static void __match_proto__(vfp_begin_f)
-vfp_testgzip_begin(struct busyobj *bo, size_t estimate)
-{
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	(void)estimate;
-	bo->vgz_rx = VGZ_NewUngzip(bo->vsl, "u F -");
-	CHECK_OBJ_NOTNULL(bo->vgz_rx, VGZ_MAGIC);
-	XXXAZ(vgz_getmbuf(bo->vgz_rx));
-}
-
-static int __match_proto__(vfp_bytes_f)
-vfp_testgzip_bytes(struct busyobj *bo, struct http_conn *htc, ssize_t bytes)
+enum vfp_status __match_proto__(vfp_pull_f)
+vfp_testgunzip_pull(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 {
 	struct vgz *vg;
-	ssize_t l, wl;
-	int i = -100;
-	size_t dl;
+	enum vgzret_e vr = VGZ_ERROR;
 	const void *dp;
-	struct storage *st;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vg = bo->vgz_rx;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	AZ(vg->vz.avail_in);
-	while (bytes > 0) {
-		st = VFP_GetStorage(bo, 0);
-		if (st == NULL)
-			return(-1);
-		l = st->space - st->len;
-		if (l > bytes)
-			l = bytes;
-		wl = htc->read(htc, st->ptr + st->len, l);
-		if (wl <= 0)
-			return (wl);
-		bytes -= wl;
-		VGZ_Ibuf(vg, st->ptr + st->len, wl);
-		VBO_extend(bo, wl);
-
-		while (!VGZ_IbufEmpty(vg)) {
+	size_t dl;
+	enum vfp_status vp;
+
+        CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	if (p == vfp_init) {
+		vg = VGZ_NewUngzip(bo->vsl, "u F -");
+		XXXAZ(vgz_getmbuf(vg));
+		*priv = (uintptr_t)vg;
+		AZ(vg->m_len);
+		return (VFP_OK);
+	}
+	if (p == vfp_fini) {
+		if (*priv != 0) {
+			CAST_OBJ_NOTNULL(vg, (void*)(*priv), VGZ_MAGIC);
+			*priv = 0;
+			(void)VGZ_Destroy(&vg);
+		}
+		return (VFP_ERROR);
+	}
+        AN(p);
+        AN(lp);
+        AN(priv);
+	CAST_OBJ_NOTNULL(vg, (void*)(*priv), VGZ_MAGIC);
+	vp = VFP_Suck(bo, p, lp);
+	if (vp == VFP_ERROR)
+		return (vp);
+	if (*lp > 0 || vp == VFP_END) {
+		VGZ_Ibuf(vg, p, *lp);
+		do {
 			VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
-			i = VGZ_Gunzip(vg, &dp, &dl);
-			if (i == VGZ_END && !VGZ_IbufEmpty(vg))
+			vr = VGZ_Gunzip(vg, &dp, &dl);
+			if (vr == VGZ_END && !VGZ_IbufEmpty(vg))
 				return(VFP_Error(bo, "Junk after gzip data"));
-			if (i != VGZ_OK && i != VGZ_END)
-				return(VFP_Error2(bo,
-				    "Invalid Gzip data", vg->vz.msg));
-		}
+			if (vr < VGZ_OK)
+				return (VFP_Error(bo,
+				    "Invalid Gzip data: %s", vg->vz.msg));
+		} while (!VGZ_IbufEmpty(vg));
 	}
-	assert(i == VGZ_OK || i == VGZ_END);
-	return (1);
-}
-
-static int __match_proto__(vfp_end_f)
-vfp_testgzip_end(struct busyobj *bo)
-{
-	struct vgz *vg;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	vg = bo->vgz_rx;
-	bo->vgz_rx = NULL;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	if (bo->state == BOS_FAILED) {
-		(void)VGZ_Destroy(&vg);
-		return(0);
+	if (vp == VFP_END) {
+		if (vr != VGZ_END)
+			return (VFP_Error(bo, "tGunzip failed"));
+		VGZ_UpdateObj(vg, bo->fetch_obj);
 	}
-	VGZ_UpdateObj(vg, bo->fetch_obj);
-	if (VGZ_Destroy(&vg) != VGZ_END)
-		return(VFP_Error(bo, "TestGunzip error at the very end"));
-	return (0);
+	return (vp);
 }
-
-const struct vfp vfp_testgzip = {
-        .begin  =       vfp_testgzip_begin,
-        .bytes  =       vfp_testgzip_bytes,
-        .end    =       vfp_testgzip_end,
-};
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 3e16bcb..1bf8bd1 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -69,22 +69,35 @@ vbf_fetch_number(const char *nbr, int radix)
 
 /*--------------------------------------------------------------------*/
 
-static int
-vbf_fetch_straight(struct busyobj *bo, struct http_conn *htc, ssize_t cl)
+static enum vfp_status __match_proto__(vfp_pull_f)
+v1f_pull_straight(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 {
-	int i;
-
-	assert(htc->body_status == BS_LENGTH);
-
-	if (cl < 0) {
-		return (VFP_Error(bo, "straight length field bogus"));
-	} else if (cl == 0)
-		return (0);
+	ssize_t l, lr;
 
-	i = bo->vfp->bytes(bo, htc, cl);
-	if (i <= 0)
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	if (p == vfp_init)
+		return (VFP_OK);
+	if (p == vfp_fini)
+		return (VFP_ERROR);
+	AN(p);
+	AN(lp);
+	AN(priv);
+
+	l = *lp;
+	*lp = 0;
+
+	if (!*priv)		// XXX: Optimize Content-Len: 0 out earlier
+		return (VFP_END);
+	if (*priv < l)
+		l = *priv;
+	lr = HTTP1_Read(&bo->htc, p, l);
+	if (lr <= 0)
 		return (VFP_Error(bo, "straight insufficient bytes"));
-	return (0);
+	*lp = lr;
+	*priv -= lr;
+	if (*priv == 0)
+		return (VFP_END);
+	return (VFP_OK);
 }
 
 /*--------------------------------------------------------------------
@@ -93,29 +106,38 @@ vbf_fetch_straight(struct busyobj *bo, struct http_conn *htc, ssize_t cl)
  * XXX: Reading one byte at a time is pretty pessimal.
  */
 
-static int
-vbf_fetch_chunked(struct busyobj *bo, struct http_conn *htc)
+static enum vfp_status __match_proto__(vfp_pull_f)
+v1f_pull_chunked(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 {
 	int i;
 	char buf[20];		/* XXX: 20 is arbitrary */
 	unsigned u;
-	ssize_t cl;
+	ssize_t cl, l, lr;
 
-	assert(htc->body_status == BS_CHUNKED);
-	do {
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	if (p == vfp_init)
+		return (VFP_OK);
+	if (p == vfp_fini)
+		return (VFP_ERROR);
+	AN(p);
+	AN(lp);
+	AN(priv);
+	l = *lp;
+	*lp = 0;
+	if (*priv == -1) {
 		/* Skip leading whitespace */
 		do {
-			if (HTTP1_Read(htc, buf, 1) <= 0)
+			if (HTTP1_Read(&bo->htc, buf, 1) <= 0)
 				return (VFP_Error(bo, "chunked read err"));
 		} while (vct_islws(buf[0]));
 
 		if (!vct_ishex(buf[0]))
-			return (VFP_Error(bo, "chunked header non-hex"));
+			 return (VFP_Error(bo, "chunked header non-hex"));
 
 		/* Collect hex digits, skipping leading zeros */
 		for (u = 1; u < sizeof buf; u++) {
 			do {
-				if (HTTP1_Read(htc, buf + u, 1) <= 0)
+				if (HTTP1_Read(&bo->htc, buf + u, 1) <= 0)
 					return (VFP_Error(bo,
 					    "chunked read err"));
 			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
@@ -128,40 +150,98 @@ vbf_fetch_chunked(struct busyobj *bo, struct http_conn *htc)
 
 		/* Skip trailing white space */
 		while(vct_islws(buf[u]) && buf[u] != '\n')
-			if (HTTP1_Read(htc, buf + u, 1) <= 0)
+			if (HTTP1_Read(&bo->htc, buf + u, 1) <= 0)
 				return (VFP_Error(bo, "chunked read err"));
 
 		if (buf[u] != '\n')
 			return (VFP_Error(bo,"chunked header no NL"));
 
 		buf[u] = '\0';
+
 		cl = vbf_fetch_number(buf, 16);
 		if (cl < 0)
 			return (VFP_Error(bo,"chunked header number syntax"));
-
-		if (cl > 0 && bo->vfp->bytes(bo, htc, cl) <= 0)
-			return (VFP_Error(bo, "chunked read err"));
-
-		i = HTTP1_Read(htc, buf, 1);
-		if (i <= 0)
-			return (VFP_Error(bo, "chunked read err"));
-		if (buf[0] == '\r' && HTTP1_Read( htc, buf, 1) <= 0)
-			return (VFP_Error(bo, "chunked read err"));
-		if (buf[0] != '\n')
-			return (VFP_Error(bo,"chunked tail no NL"));
-	} while (cl > 0);
-	return (0);
+		*priv = cl;
+	}
+	if (*priv > 0) {
+		if (*priv < l)
+			l = *priv;
+		lr = HTTP1_Read(&bo->htc, p, l);
+		if (lr <= 0)
+			return (VFP_Error(bo, "straight insufficient bytes"));
+		*lp = lr;
+		*priv -= lr;
+		if (*priv == 0)
+			*priv = -1;
+		return (VFP_OK);
+	}
+	AZ(*priv);
+	i = HTTP1_Read(&bo->htc, buf, 1);
+	if (i <= 0)
+		return (VFP_Error(bo, "chunked read err"));
+	if (buf[0] == '\r' && HTTP1_Read(&bo->htc, buf, 1) <= 0)
+		return (VFP_Error(bo, "chunked read err"));
+	if (buf[0] != '\n')
+		return (VFP_Error(bo,"chunked tail no NL"));
+	return (VFP_END);
 }
 
 /*--------------------------------------------------------------------*/
 
-static void
-vbf_fetch_eof(struct busyobj *bo, struct http_conn *htc)
+static enum vfp_status __match_proto__(vfp_pull_f)
+v1f_pull_eof(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 {
+	ssize_t l, lr;
 
-	assert(htc->body_status == BS_EOF);
-	if (bo->vfp->bytes(bo, htc, SSIZE_MAX) < 0)
-		(void)VFP_Error(bo,"eof socket fail");
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	if (p == vfp_init)
+		return (VFP_OK);
+	if (p == vfp_fini)
+		return (VFP_ERROR);
+	AN(p);
+	AN(lp);
+	AN(priv);
+
+	l = *lp;
+	*lp = 0;
+	lr = HTTP1_Read(&bo->htc, p, l);
+	if (lr < 0)
+		return (VFP_Error(bo,"eof socket fail"));
+	if (lr == 0)
+		return (VFP_END);
+	*lp = lr;
+	return (VFP_OK);
+}
+
+/*--------------------------------------------------------------------
+ */
+
+ssize_t
+V1F_Setup_Fetch(struct busyobj *bo)
+{
+	struct http_conn *htc;
+	ssize_t cl;
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	htc = &bo->htc;
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC);
+
+	switch(htc->body_status) {
+	case BS_EOF:
+		VFP_Push(bo, v1f_pull_eof, 0);
+		return(-1);
+	case BS_LENGTH:
+		cl = vbf_fetch_number(bo->h_content_length, 10);
+		VFP_Push(bo, v1f_pull_straight, cl);
+		return (cl);
+	case BS_CHUNKED:
+		VFP_Push(bo, v1f_pull_chunked, -1);
+		return (-1);
+	default:
+		break;
+	}
+	return (-1);
 }
 
 /*--------------------------------------------------------------------
@@ -267,6 +347,8 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	HTTP1_Init(htc, bo->ws, vc->fd, vc->vsl,
 	    cache_param->http_resp_size,
 	    cache_param->http_resp_hdr_len);
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	CHECK_OBJ_NOTNULL(&bo->htc, HTTP_CONN_MAGIC);
 
 	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
 
@@ -308,81 +390,3 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	return (0);
 }
 
-/*--------------------------------------------------------------------
- * This function is either called by the requesting thread OR by a
- * dedicated body-fetch work-thread.
- *
- * We get passed the busyobj in the priv arg, and we inherit a
- * refcount on it, which we must release, when done fetching.
- */
-
-void
-V1F_fetch_body(struct busyobj *bo)
-{
-	struct storage *st;
-	ssize_t cl;
-	struct http_conn *htc;
-	struct object *obj;
-
-	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	htc = &bo->htc;
-	CHECK_OBJ_ORNULL(bo->vbc, VBC_MAGIC);
-	obj = bo->fetch_obj;
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
-	AN(bo->vbc);
-
-	assert(bo->state == BOS_FETCHING);
-
-	AN(bo->vfp);
-	AZ(bo->vgz_rx);
-	assert(VTAILQ_EMPTY(&obj->store));
-
-	/* XXX: pick up estimate from objdr ? */
-	cl = 0;
-	switch (htc->body_status) {
-	case BS_LENGTH:
-		cl = vbf_fetch_number(bo->h_content_length, 10);
-
-		bo->vfp->begin(bo, cl);
-		if (bo->state == BOS_FETCHING && cl > 0)
-			bo->should_close |= vbf_fetch_straight(bo, htc, cl);
-		if (bo->vfp->end(bo))
-			assert(bo->state == BOS_FAILED);
-		break;
-	case BS_CHUNKED:
-		bo->vfp->begin(bo, cl > 0 ? cl : 0);
-		if (bo->state == BOS_FETCHING)
-			bo->should_close |= vbf_fetch_chunked(bo, htc);
-		if (bo->vfp->end(bo))
-			assert(bo->state == BOS_FAILED);
-		break;
-	case BS_EOF:
-		bo->vfp->begin(bo, cl > 0 ? cl : 0);
-		if (bo->state == BOS_FETCHING)
-			vbf_fetch_eof(bo, htc);
-		bo->should_close = 1;
-		if (bo->vfp->end(bo))
-			assert(bo->state == BOS_FAILED);
-		break;
-	default:
-		WRONG("Wrong body_status");
-	}
-	AZ(bo->vgz_rx);
-
-	/*
-	 * Trim or delete the last segment, if any
-	 */
-
-	st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
-	/* XXX: Temporary:  Only trim if we are not streaming */
-	if (st != NULL && !bo->do_stream) {
-		/* XXX: is any of this safe under streaming ? */
-		if (st->len == 0) {
-			VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
-			STV_free(st);
-		} else if (st->len < st->space) {
-			STV_trim(st, st->len, 1);
-		}
-	}
-}
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 4aa08ce..1a05479 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -296,12 +296,11 @@ VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
  */
 
 void
-VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
+VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap)
 {
 	char *p;
 	const char *u, *f;
 	unsigned n, mlen;
-	va_list ap;
 	txt t;
 
 	AN(fmt);
@@ -329,9 +328,7 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
 		VSL_Flush(vsl, 1);
 
 	p = VSL_DATA(vsl->wlp);
-	va_start(ap, fmt);
 	n = vsnprintf(p, mlen, fmt, ap);
-	va_end(ap);
 	if (n > mlen - 1)
 		n = mlen - 1;	/* we truncate long fields */
 	p[n++] = '\0';		/* NUL-terminated */
@@ -343,6 +340,16 @@ VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
 		VSL_Flush(vsl, 0);
 }
 
+void
+VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
+{
+	va_list ap;
+
+	va_start(ap, fmt);
+	VSLbv(vsl, tag, fmt, ap);
+	va_end(ap);
+}
+
 /*--------------------------------------------------------------------
  * Setup a VSL buffer, allocate space if none provided.
  */



From phk at FreeBSD.org  Sat Dec  7 11:03:07 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 07 Dec 2013 12:03:07 +0100
Subject: [master] af79dc8 Fix this testcase to not depend on param
 "accept_filter" to exist.
Message-ID: <E1VpFf9-0005Cq-Tk@project.varnish-software.com>


commit af79dc8504b6b036ef4c182afa3d6723fe6953e9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Dec 7 11:02:51 2013 +0000

    Fix this testcase to not depend on param "accept_filter" to exist.

diff --git a/bin/varnishtest/tests/b00013.vtc b/bin/varnishtest/tests/b00013.vtc
index d48cd5a..ffae946 100644
--- a/bin/varnishtest/tests/b00013.vtc
+++ b/bin/varnishtest/tests/b00013.vtc
@@ -9,7 +9,11 @@ server s1 {
 	txresp -body "foobar"
 } -start
 
-varnish v1 -arg "-p accept_filter=false" -vcl+backend {} -start
+varnish v1 -vcl+backend {}
+
+# NB: The accept_filter param may not exist.
+varnish v1 -cli "param.set accept_filter false"
+varnish v1 -start
 
 client c1 {
 	send "GET /foo HTTP/1.1\r\n\r\nGET "



From tfheen at varnish-software.com  Mon Dec  9 12:06:57 2013
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Mon, 09 Dec 2013 13:06:57 +0100
Subject: [3.0] dd0e696 Document %D and %T for varnishncsa
Message-ID: <E1Vpzc1-0000S3-NA@project.varnish-software.com>


commit dd0e69604c26a9a7852d7804d6e75111e06cf6ee
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu May 2 14:04:22 2013 +0200

    Document %D and %T for varnishncsa

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 2937b26..8cd7b05 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -55,6 +55,9 @@ The following options are available:
    	         In CLF format, i.e. a '-' rather than a 0 when no
    	         bytes are sent.
 
+	      %D
+	         Time taken to serve the request, in microseconds.
+
 	      %H 
 	         The request protocol. Defaults to HTTP/1.0 if not
                  known.
@@ -95,6 +98,9 @@ The following options are available:
 		 specified by X.  The time specification format is the
 		 same as for strftime(3).
 
+	      %T
+	         Time taken to serve the request, in seconds.
+
 	      %U
 	         The request URL without any query string. Defaults to
                  '-' if not known.



From martin at varnish-software.com  Mon Dec  9 12:21:51 2013
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 09 Dec 2013 13:21:51 +0100
Subject: [master] 46c0c37 Fix wrong argument order when picking the Host
 header.
Message-ID: <E1VpzqR-00024s-QI@project.varnish-software.com>


commit 46c0c379572d5dee5bc345f3aaf3bf83b16eac39
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Dec 9 12:41:33 2013 +0100

    Fix wrong argument order when picking the Host header.
    
    Fix wrong argument order for isprefix() when picking the Host
    header. This caused the header to never be matched.
    
    Fixes: #1383

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 452cc70..b97d667 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -776,7 +776,7 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				    0, NULL);
 				break;
 			case SLT_ReqHeader:
-				if (isprefix(b, e, "Host:", &p))
+				if (isprefix(b, "Host:", e, &p))
 					frag_line(p, e, &CTX.frag[F_host]);
 				else if (isprefix(b, "Authorization:", e, &p) &&
 				    isprefix(p, "basic", e, &p))



From martin at varnish-software.com  Mon Dec  9 12:21:51 2013
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 09 Dec 2013 13:21:51 +0100
Subject: [master] 319cb74 Remove log record terminating null char before
 processing log lines.
Message-ID: <E1VpzqR-000255-VK@project.varnish-software.com>


commit 319cb7442fc504e7b43d5f4e834265b700eed7bb
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Dec 9 12:58:43 2013 +0100

    Remove log record terminating null char before processing log lines.
    
    Fix the end of log record pointer to not include the record
    terminating null char. This caused the null to be part of the captured
    fragments and sent to the output.
    
    Fixes: #1382

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index b97d667..c1c1e36 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -746,6 +746,8 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			tag = VSL_TAG(t->c->rec.ptr);
 			b = VSL_CDATA(t->c->rec.ptr);
 			e = b + VSL_LEN(t->c->rec.ptr);
+			while (e > b && e[-1] == '\0')
+				e--;
 
 			switch (tag) {
 			case SLT_ReqStart:



From tfheen at varnish-software.com  Mon Dec  9 12:28:04 2013
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Mon, 09 Dec 2013 13:28:04 +0100
Subject: [3.0] fc26e3f doc typo
Message-ID: <E1VpzwS-0002k7-9k@project.varnish-software.com>


commit fc26e3f609b6d0e2c035c8d7b68dd4efb6bbff42
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Dec 9 13:27:27 2013 +0100

    doc typo

diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst
index 5e62e77..29ca9cb 100644
--- a/doc/sphinx/installation/help.rst
+++ b/doc/sphinx/installation/help.rst
@@ -39,7 +39,7 @@ it mostly on topic, and dont paste random links unless they are
 Mailing Lists
 =============
 
-Getting on or off our mailinglist happens through MailMan_.
+Getting on or off our mailinglist happens through Mailman_.
 
 If you are going to use Varnish, subscribing to our ``varnish-announce``
 mailing list is probably a very good idea.  The typical pattern is that



From phk at FreeBSD.org  Mon Dec  9 13:09:34 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 09 Dec 2013 14:09:34 +0100
Subject: [master] 3be8ba2 Create .rst documentation from the vmod.vcc file.
Message-ID: <E1Vq0ac-0001w7-3n@project.varnish-software.com>


commit 3be8ba2de0cd9d6e35f181358e08d804f78f1b26
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 9 13:01:21 2013 +0000

    Create .rst documentation from the vmod.vcc file.
    
    This causes a change of syntax for the vcc file, it is however
    very slight:  "Module", "Init", "Function", "Object" and "Method" needs
    a "$" prefix, and the methods refer to the previous $Objecet without
    the enclosing { }.
    
    The first contiguous set of lines starting with "#" is treated as
    a copyright notice, and emitted last, unless the first line is "#-"

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 3190185..af52ec5 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -92,13 +92,16 @@ class token(object):
 #######################################################################
 
 class vmod(object):
-	def __init__(self, nam):
+	def __init__(self, nam, dnam):
 		if not is_c_name(nam):
 			raise Exception("Module name '%s' is illegal" % nam)
 		self.nam = nam
+		self.dnam = dnam
 		self.init = None
 		self.funcs = list()
 		self.objs = list()
+		self.doc_str = []
+		self.doc_order = []
 
 	def set_init(self, nam):
 		if self.init != None:
@@ -109,13 +112,15 @@ class vmod(object):
 
 	def add_func(self, fn):
 		self.funcs.append(fn)
+		self.doc_order.append(fn)
 
 	def add_obj(self, obj):
 		self.objs.append(obj)
-		obj.set_modnam(self.nam)
+		self.doc_order.append(obj)
 
 	def c_proto(self, fo):
 		for o in self.objs:
+			o.fixup(self.nam)
 			o.c_proto(fo)
 			fo.write("\n")
 		for f in self.funcs:
@@ -235,6 +240,37 @@ class vmod(object):
 		s += "};\n"
 		return s
 
+	def doc(self, l):
+		self.doc_str.append(l)
+
+	def doc_dump(self, fo, suf):
+		i = "vmod_" + self.nam + " -- " + self.dnam
+		fo.write("=" * len(i) + "\n")
+		fo.write(i + "\n")
+		fo.write("=" * len(i) + "\n")
+		fo.write("\n")
+		fo.write("SYNOPSIS\n")
+		fo.write("========\n")
+		fo.write("\n")
+		fo.write("import %s [from \"path\"] ;\n" % self.nam)
+		fo.write("\n")
+		for i in self.doc_str:
+			fo.write(i + "\n")
+		fo.write("CONTENTS\n")
+		fo.write("========\n")
+		fo.write("\n")
+		l = []
+		for i in self.funcs:
+			l.append(i.doc_idx(suf))
+		for i in self.objs:
+			l += i.doc_idx(suf)
+		l.sort()
+		for i in l:
+			fo.write("* " + i[1] + "\n")
+		fo.write("\n")
+		for i in self.doc_order:
+			i.doc_dump(fo)
+
 #######################################################################
 
 class func(object):
@@ -249,6 +285,7 @@ class func(object):
 		self.al = al
 		self.retval = retval
 		self.pfx = None
+		self.doc_str = []
 
 	def __repr__(self):
 		return "<FUNC %s %s>" % (self.retval, self.nam)
@@ -309,6 +346,43 @@ class func(object):
 			s += a.c_strspec()
 		return s
 
+	def doc(self, l):
+		self.doc_str.append(l)
+
+	def doc_proto(self):
+		s = self.retval + " " + self.nam + "("
+		d = ""
+		for i in self.al:
+			s += d + i.typ
+			d = ", "
+		s += ")"
+		return s
+
+	def doc_idx(self, suf):
+		if suf == "":
+			return (self.nam, ":ref:`func_" + self.nam + "`")
+		else:
+			return (self.nam, self.doc_proto())
+
+	def doc_dump(self, fo):
+		s = self.doc_proto()
+		fo.write(".. _func_" + self.nam + ":\n\n")
+		fo.write(s + "\n")
+		fo.write("-" * len(s) + "\n")
+		fo.write("\n")
+		fo.write("Prototype\n")
+		s = "\t" + self.retval + " " + self.nam + "("
+		d = ""
+		for i in self.al:
+			s += d + i.typ
+			if i.nam != None:
+				s += " " + i.nam
+			d = ", "
+		fo.write(s + ")\n")
+		for i in self.doc_str:
+			fo.write(i + "\n")
+		
+
 #######################################################################
 
 class obj(object):
@@ -317,8 +391,10 @@ class obj(object):
 		self.init = None
 		self.fini = None
 		self.methods = list()
+		self.doc_str = []
 
-	def set_modnam(self, modnam):
+	def fixup(self, modnam):
+		assert self.nam != None
 		self.st = "struct vmod_" + modnam + "_" + self.nam
 		self.init.set_pfx(self.st + " **, const char *")
 		self.fini.set_pfx(self.st + " **")
@@ -378,6 +454,32 @@ class obj(object):
 		s += '\t\t"\\0",\n'
 		return s
 
+	def doc(self, l):
+		self.doc_str.append(l)
+
+	def doc_idx(self, suf):
+		l = []
+		if suf == "":
+			l.append((self.nam, ":ref:`obj_" + self.nam + "`"))
+		else:
+			l.append((self.nam, "Object " + self.nam))
+		for i in self.methods:
+			l.append(i.doc_idx(suf))
+		return l
+
+	def doc_dump(self, fo):
+		fo.write(".. _obj_" + self.nam + ":\n\n")
+		s = "Object " + self.nam
+		fo.write(s + "\n")
+		fo.write("=" * len(s) + "\n")
+		fo.write("\n")
+
+		for i in self.doc_str:
+			fo.write(i + "\n")
+
+		for i in self.methods:
+			i.doc_dump(fo)
+
 #######################################################################
 
 class arg(object):
@@ -397,42 +499,28 @@ class arg(object):
 		return "??"
 
 #######################################################################
-
-f = open(specfile, "r")
-tl = list()
-lines = list()
-ln = 0
-for l in f:
-	ln += 1
-	lines.append(l)
-	if l == "":
-		continue
-	l = re.sub("[ \t]*#.*$", "", l)
-	l = re.sub("[ \t]*\n", "", l)
-	l = re.sub("([(){},])", r' \1 ', l)
-	if l == "":
-		continue
-	for j in l.split():
-		tl.append(token(ln, 0, j))
-f.close()
-
-#######################################################################
 #
 #
 def parse_enum2(tl):
-	t = tl.pop(0)
+	t = tl.get_token()
 	if t.str != "{":
 		raise Exception("expected \"{\"")
 	s = "ENUM\\0"
+	t = None
 	while True:
-		t = tl.pop(0)
+		if t == None:
+			t = tl.get_token()
 		if t.str == "}":
 			break
 		s += t.str + "\\0"
-		if tl[0].str == ",":
-			tl.pop(0)
-		elif tl[0].str != "}":
-			raise Exception("Expceted \"}\" or \",\"")
+		t = tl.get_token()
+		if t.str == ",":
+			t = None
+		elif t.str == "}":
+			break
+		else:
+			raise Exception(
+			    "Expected \"}\" or \",\" not \"%s\"" % t.str)
 	s += "\\0"
 	return arg("ENUM", det=s)
 
@@ -440,46 +528,67 @@ def parse_enum2(tl):
 #
 #
 
+def parse_module(tl):
+	nm = tl.get_token().str
+	s = ""
+	while len(tl.tl) > 0:
+		s += " " + tl.get_token().str
+	dnm = s[1:]
+	return vmod(nm, dnm)
+
+#######################################################################
+#
+#
+
 def parse_func(tl, rt_type = None, obj=None):
 	al = list()
 	if rt_type == None:
-		t = tl.pop(0)
+		t = tl.get_token()
 		rt_type = t.str
 	if rt_type not in ctypes:
 		raise Exception(
 		    "Return type '%s' not a valid type" % rt_type)
 
-	t = tl.pop(0)
+	t = tl.get_token()
 	fname = t.str
 	if obj != None and fname[0] == "." and is_c_name(fname[1:]):
 		fname = obj + fname
 	elif not is_c_name(fname):
 		raise Exception("Function name '%s' is illegal" % fname)
 
-	t = tl.pop(0)
+	t = tl.get_token()
 	if t.str != "(":
 		raise Exception("Expected \"(\" got \"%s\"", t.str)
 
+	t = None
 	while True:
-		t = tl.pop(0)
-		if t.str == ")":
-			break
+		if t == None:
+			t = tl.get_token()
+		assert t != None
+
 		if t.str == "ENUM":
 			al.append(parse_enum2(tl))
 		elif t.str in ctypes:
 			al.append(arg(t.str))
+		elif t.str == ")":
+			break
 		else:
 			raise Exception("ARG? %s" % t.str)
-		if is_c_name(tl[0].str):
-			al[-1].nam = tl[0].str
-			t = tl.pop(0)
-		if tl[0].str == ",":
-			tl.pop(0)
-		elif tl[0].str != ")":
-			raise Exception("Expceted \")\" or \",\"")
+		t = tl.get_token()
+		if is_c_name(t.str):
+			al[-1].nam = t.str
+			t = None
+		elif t.str == ",":
+			t = None
+		elif t.str == ")":
+			break
+		else:
+			raise Exception(
+			    "Expceted \")\" or \",\" not \"%s\"" % t.str)
 	if t.str != ")":
 		raise Exception("End Of Input looking for ')'")
 	f = func(fname, rt_type, al)
+
 	return f
 
 #######################################################################
@@ -487,48 +596,166 @@ def parse_func(tl, rt_type = None, obj=None):
 #
 
 def parse_obj(tl):
-	o = obj(tl[0].str)
 	f = parse_func(tl, "VOID")
+	o = obj(f.nam)
 	o.set_init(f)
-	t = tl.pop(0)
-	assert t.str == "{"
-	while True:
-		t = tl.pop(0)
-		if t.str == "}":
-			break
-		assert t.str == "Method"
-		f = parse_func(tl, obj=o.nam)
-		o.add_method(f)
 	return o
 
+
+#######################################################################
+# A section of the specfile, starting at a keyword
+
+class file_section(object):
+	def __init__(self):
+		self.l = []
+		self.tl = []
+
+	def add_line(self, ln, l):
+		self.l.append((ln, l))
+
+	def get_token(self):
+		while True:
+			if len(self.tl) > 0:
+				# print("T\t", self.tl[0])
+				return self.tl.pop(0)
+			if len(self.l) == 0:
+				break
+			self.more_tokens()
+		return None
+
+	def more_tokens(self):
+		ln,l = self.l.pop(0)
+		if l == "":
+			return
+		l = re.sub("[ \t]*#.*$", "", l)
+		l = re.sub("[ \t]*\n", "", l)
+		l = re.sub("([(){},])", r' \1 ', l)
+		if l == "":
+			return
+		for j in l.split():
+			self.tl.append(token(ln, 0, j))
+
+	def parse(self, vx):
+		t = self.get_token()
+		if t == None:
+			return
+		t0 = t.str
+		if t.str == "$Module":
+			o = parse_module(self)
+			vx.append(o)
+		elif t.str == "$Init":
+			x = self.get_token()
+			vx[0].set_init(x.str)
+			o = None
+		elif t.str == "$Function":
+			if len(vx) == 2:
+				vx.pop(-1)
+			o = parse_func(self)
+			vx[0].add_func(o)
+		elif t.str == "$Object":
+			if len(vx) == 2:
+				vx.pop(-1)
+			o = parse_obj(self)
+			vx[0].add_obj(o)
+			vx.append(o)
+		elif t.str == "$Method":
+			if len(vx) != 2:
+				raise Exception("$Method outside $Object")
+			o = parse_func(self, obj = vx[1].nam)
+			vx[1].add_method(o)
+		else:
+			raise Exception("Unknown keyword: " + t.str)
+		assert len(self.tl) == 0
+		if o == None:
+			print("NB:")
+			print("%s description is not included in .rst:" %t0)
+			for ln,i in self.l:
+				print("\t", i)
+		else:
+			for ln,i in self.l:
+				o.doc(i)
+
 #######################################################################
-# The first thing in the file must be the Module declaration
+# Polish the copyright message
 #
+def polish(l):
+	if len(l[0]) == 0:
+		l.pop(0)
+		return True
+	c = l[0][0]
+	for i in l:
+		if len(i) == 0:
+			continue
+		if i[0] != c:
+			c = None
+			break
+	if c != None:
+		for i in range(len(l)):
+			l[i] = l[i][1:]
+		return True
+	return False
 
-t = tl.pop(0)
-if t.str != "Module":
-	raise Exception("\"Module\" must be first in file")
-t = tl.pop(0)
-vmod = vmod(t.str)
+#######################################################################
+# Read the file in
+
+f = open(specfile, "r")
+lines = []
+for i in f:
+	lines.append(i.rstrip())
+f.close()
+ln = 0
 
 #######################################################################
-# Parse the rest of the file
-#
+# First collect the copyright:  All initial lines starting with '#'
+
+copyright = []
+while len(lines[0]) > 0 and lines[0][0] == "#":
+	ln += 1
+	copyright.append(lines.pop(0))
 
-while len(tl) > 0:
-	t = tl.pop(0)
-
-	if t.str == "Init":
-		t = tl.pop(0)
-		vmod.set_init(t.str)
-	elif t.str == "Function":
-		f = parse_func(tl)
-		vmod.add_func(f)
-	elif t.str == "Object":
-		o = parse_obj(tl)
-		vmod.add_obj(o)
+if len(copyright) > 0:
+	if copyright[0] == "#-":
+		copyright = [ ]
 	else:
-		raise Exception("Expected \"Init\", \"Fini\" or \"Function\"")
+		while polish(copyright):
+			continue
+
+if False:
+	for i in copyright:
+		print("(C)\t", i)
+
+#######################################################################
+# Break into sections
+
+keywords = {
+	"$Module":	True,
+	"$Function":	True,
+	"$Object":	True,
+	"$Method":	True,
+	"$Init":	True,
+}
+
+sl = []
+sc = file_section()
+sl.append(sc)
+while len(lines) > 0:
+	ln += 1
+	l = lines.pop(0)
+	j = l.split()
+	if len(j) > 0 and j[0] in keywords:
+		sc = file_section()
+		sl.append(sc)
+	sc.add_line(ln,l)
+
+#######################################################################
+# Parse each section
+
+first = True
+
+vx = []
+for i in sl:
+	i.parse(vx)
+	assert len(i.tl) == 0
 
 #######################################################################
 # Parsing done, now process
@@ -545,7 +772,7 @@ fh.write('struct VCL_conf;\n')
 fh.write('struct vmod_priv;\n')
 fh.write("\n");
 
-vmod.c_proto(fh)
+vx[0].c_proto(fh)
 
 fc.write("""#include "config.h"
 
@@ -556,8 +783,23 @@ fc.write("""#include "config.h"
 
 """)
 
-vmod.c_typedefs(fc)
-vmod.c_vmod(fc)
+vx[0].c_typedefs(fc)
+vx[0].c_vmod(fc)
 
 fc.close()
 fh.close()
+
+for suf in ("", ".man"):
+	fr = open("vmod_" + vx[0].nam + suf + ".rst", "w")
+	vx[0].doc_dump(fr, suf)
+
+	if len(copyright) > 0:
+		fr.write("\n")
+		fr.write("COPYRIGHT\n")
+		fr.write("=========\n")
+		fr.write("\n::\n\n")
+		for i in copyright:
+			fr.write("  " + i + "\n")
+		fr.write("\n")
+
+	fr.close()
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 993c1bc..69bfb38 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -1,4 +1,4 @@
-#-
+#
 # Copyright (c) 2010-2013 Varnish Software AS
 # All rights reserved.
 #
@@ -25,15 +25,49 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-Module debug
-Init init_function
-Function VOID panic(STRING_LIST)
-Function STRING author(ENUM { phk, des, kristian, mithrandir })
-Function VOID test_priv_call(PRIV_CALL)
-Function VOID test_priv_vcl(PRIV_VCL)
-Object obj(STRING) {
-	# NOTE: .enum before .foo as part of test r01332.vtc
-	Method VOID .enum(ENUM { phk, des, kristian, mithrandir, martin })
-	Method STRING .foo(STRING why)
-	Method TIME .date()
-}
+$Module debug Development, test and debug
+
+DESCRIPTION
+===========
+
+This vmod is used to develop, test and debug the various aspects
+of VMOD handling in Varnish.
+
+$Init init_function
+
+Call this whenever a VCL is loaded which imports this vmod.
+
+$Function VOID panic(STRING_LIST)
+
+Don't.
+
+$Function STRING author(ENUM { phk, des, kristian, mithrandir })
+
+Test function for ENUM arguments
+
+$Function VOID test_priv_call(PRIV_CALL)
+
+Test function for call private pointers
+
+$Function VOID test_priv_vcl(PRIV_VCL)
+
+Test function for VCL private pointers
+
+$Object obj(STRING) 
+
+Test object
+
+.. NOTE: .enum before .foo as part of test r01332.vtc
+$Method VOID .enum(ENUM { phk, des, kristian, mithrandir, martin })
+
+Testing that emums work as part of object and that the parser isn't
+(too) buggy.
+
+$Method STRING .foo(STRING why)
+
+Foo indeed.
+
+$Method TIME .date()
+
+You never know when you need a date.
+
diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 8b61944..37d8f56 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -25,24 +25,20 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-Module directors
+$Module directors Backend traffic directors
 
-Object round_robin() {
-	Method VOID .add_backend(BACKEND)
-	Method BACKEND .backend()
-}
+$Object round_robin()
+$Method VOID .add_backend(BACKEND)
+$Method BACKEND .backend()
 
-Object fallback() {
-	Method VOID .add_backend(BACKEND)
-	Method BACKEND .backend()
-}
+$Object fallback()
+$Method VOID .add_backend(BACKEND)
+$Method BACKEND .backend()
 
-Object random() {
-	Method VOID .add_backend(BACKEND, REAL)
-	Method BACKEND .backend()
-}
+$Object random()
+$Method VOID .add_backend(BACKEND, REAL)
+$Method BACKEND .backend()
 
-Object hash() {
-	Method VOID .add_backend(BACKEND, REAL)
-	Method BACKEND .backend(STRING_LIST)
-}
+$Object hash()
+$Method VOID .add_backend(BACKEND, REAL)
+$Method BACKEND .backend(STRING_LIST)
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 7db1ab0..31d25e5 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -1,5 +1,5 @@
 #-
-# Copyright (c) 2010-2011 Varnish Software AS
+# Copyright (c) 2010-2013 Varnish Software AS
 # All rights reserved.
 #
 # Author: Poul-Henning Kamp <phk at FreeBSD.org>
@@ -25,17 +25,141 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-Module std
-Function STRING toupper(STRING_LIST)
-Function STRING tolower(STRING_LIST)
-Function VOID set_ip_tos(INT)
-Function REAL random(REAL, REAL)
-Function VOID log(STRING_LIST)
-Function VOID syslog(INT, STRING_LIST)
-Function STRING fileread(PRIV_CALL, STRING)
-Function DURATION duration(STRING, DURATION)
-Function INT integer(STRING, INT)
-Function VOID collect(HEADER)
-Function IP ip(STRING, IP)
-Function BOOL healthy(BACKEND)
-Function INT port(IP)
+$Module std Varnish Standard Module
+
+DESCRIPTION
+===========
+
+Vmod_std contains basic functions which are part and parcel of Varnish,
+but which for reasons of architecture fit better in a VMOD.
+
+One particular class of functions in vmod_std is the conversions functions
+which all have the form::
+
+	TYPE type(STRING, TYPE)
+
+These functions attempt to convert STRING to the TYPE, and if that fails,
+they return the second argument, which must have the given TYPE.
+
+$Function STRING toupper(STRING_LIST)
+
+Description
+	Converts the string *s* to upper case.
+Example
+	set beresp.http.x-scream = std.toupper("yes!");
+
+$Function STRING tolower(STRING_LIST)
+
+Description
+	Converts the string *s* to lower case.
+Example
+	set beresp.http.x-nice = std.tolower("VerY");
+
+$Function VOID set_ip_tos(INT)
+
+Description
+	Sets the Type-of-Service flag for the current session. Please
+	note that the TOS flag is not removed by the end of the
+	request so probably want to set it on every request should you
+	utilize it.
+Example
+	| if (req.url ~ ^/slow/) {
+	|    std.set_ip_tos(0x0);
+	| }
+
+$Function REAL random(REAL, REAL)
+
+Description
+	Returns a random REAL number between *a* and *b*.
+Example
+	set beresp.http.x-random-number = std.random(1, 100);
+
+$Function VOID log(STRING_LIST)
+
+Description
+	Logs *string* to the shared memory log, using VSL tag *SLT_VCL_Log*.
+Example
+	std.log("Something fishy is going on with the vhost " + req.host);
+
+$Function VOID syslog(INT, STRING_LIST)
+
+Description
+	Logs *string* to syslog marked with *priority*.  See your
+	system's syslog.h file for the legal values of *priority*.
+Example
+	std.syslog(8 + 1, "Something is wrong");
+
+$Function STRING fileread(PRIV_CALL, STRING)
+
+Description
+	Reads a file and returns a string with the content. Please
+	note that it is not recommended to send variables to this
+	function the caching in the function doesn't take this into
+	account. Also, files are not re-read.
+Example
+	set beresp.http.x-served-by = std.fileread("/etc/hostname");
+
+$Function VOID collect(HEADER)
+
+Description
+	Collapses the header, joining the headers into one.
+Example
+	std.collect(req.http.cookie);
+	This will collapse several Cookie: headers into one, long
+	cookie header.
+
+$Function DURATION duration(STRING, DURATION)
+
+Description
+	Converts the string *s* to seconds. *s* can be quantified with
+	the usual s (seconds), m (minutes), h (hours), d (days) and w
+	(weeks) units. If *s* fails to parse, *fallback* will be returned.
+Example
+	set beresp.ttl = std.duration("1w", 3600s);
+
+$Function INT integer(STRING, INT)
+
+Description
+	Converts the string *s* to an integer.  If *s* fails to parse,
+	*fallback* will be returned.
+Example
+	if (std.integer(beresp.http.x-foo, 0) > 5) { ... }
+
+$Function IP ip(STRING, IP)
+
+Description
+	Converts string *s* to the first IP number returned by
+	the system library function getaddrinfo(3).  If conversion
+	fails, *fallback* will be returned.
+Example
+	if (std.ip(req.http.X-forwarded-for, "0.0.0.0") ~ my_acl) { ... }
+
+$Function BOOL healthy(BACKEND)
+
+Description
+	Returns true if the backend is healthy
+
+$Function INT port(IP)
+
+Description
+	Returns the port number of an IP address
+
+
+SEE ALSO
+========
+
+* vcl(7)
+* varnishd(1)
+
+HISTORY
+=======
+
+The Varnish standard module was released along with Varnish Cache 3.0.
+This manual page was written by Per Buer with help from Martin Blix
+Grydeland.
+
+COPYRIGHT
+=========
+
+This document is licensed under the same licence as Varnish
+itself. See LICENCE for details.



From phk at FreeBSD.org  Mon Dec  9 13:56:48 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 09 Dec 2013 14:56:48 +0100
Subject: [master] b8034f3 Use weaker plain magic check for WS overflows to
 avoid recursive panics
Message-ID: <E1Vq1KK-0007dI-Dv@project.varnish-software.com>


commit b8034f3196c40f2f8efb837fc8ffd86d3f6388fa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 9 13:56:17 2013 +0000

    Use weaker plain magic check for WS overflows to avoid recursive panics

diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 17245b1..0ee8124 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -85,7 +85,7 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
 static void
 WS_MarkOverflow(struct ws *ws)
 {
-	WS_Assert(ws);
+	CHECK_OBJ_NOTNULL(ws, WS_MAGIC);
 
 	ws->id[0] &= ~0x40;		// Cheasy toupper()
 }
@@ -220,7 +220,7 @@ WS_ReleaseP(struct ws *ws, char *ptr)
 int
 WS_Overflowed(const struct ws *ws)
 {
-	WS_Assert(ws);
+	CHECK_OBJ_NOTNULL(ws, WS_MAGIC);
 
 	if (ws->id[0] & 0x40)
 		return (0);



From phk at varnish-cache.org  Mon Dec  9 16:43:22 2013
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 09 Dec 2013 17:43:22 +0100
Subject: [master] a7808f1 Don't attempt to dump bad WS structs
Message-ID: <E1Vq3vW-0006gZ-4E@project.varnish-software.com>


commit a7808f14993a8a47bc98933cf1786467228bf761
Author: Poul-Henning Kamp <phk at varnish-cache.org>
Date:   Mon Dec 9 17:42:43 2013 +0100

    Don't attempt to dump bad WS structs

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 97775ae..8f8f8fd 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -109,22 +109,26 @@ pan_ws(const struct ws *ws, int indent)
 {
 
 	VSB_printf(pan_vsp, "%*sws = %p {", indent, "", ws);
-	if (WS_Overflowed(ws))
-		VSB_printf(pan_vsp, " OVERFLOW");
-	VSB_printf(pan_vsp, "\n%*sid = \"%s\",\n", indent + 2, "", ws->id);
-	VSB_printf(pan_vsp, "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s);
-	if (ws->f > ws->s)
-		VSB_printf(pan_vsp, ",+%ld", (long) (ws->f - ws->s));
-	else
-		VSB_printf(pan_vsp, ",%p", ws->f);
-	if (ws->r > ws->s)
-		VSB_printf(pan_vsp, ",+%ld", (long) (ws->r - ws->s));
-	else
-		VSB_printf(pan_vsp, ",%p", ws->r);
-	if (ws->e > ws->s)
-		VSB_printf(pan_vsp, ",+%ld", (long) (ws->e - ws->s));
-	else
-		VSB_printf(pan_vsp, ",%p", ws->e);
+	if (VALID_OBJ(ws, WS_MAGIC)) {
+		if (WS_Overflowed(ws))
+			VSB_printf(pan_vsp, " OVERFLOW");
+		VSB_printf(pan_vsp, "\n%*sid = \"%s\",\n", indent + 2, "", ws->id);
+		VSB_printf(pan_vsp, "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s);
+		if (ws->f > ws->s)
+			VSB_printf(pan_vsp, ",+%ld", (long) (ws->f - ws->s));
+		else
+			VSB_printf(pan_vsp, ",%p", ws->f);
+		if (ws->r > ws->s)
+			VSB_printf(pan_vsp, ",+%ld", (long) (ws->r - ws->s));
+		else
+			VSB_printf(pan_vsp, ",%p", ws->r);
+		if (ws->e > ws->s)
+			VSB_printf(pan_vsp, ",+%ld", (long) (ws->e - ws->s));
+		else
+			VSB_printf(pan_vsp, ",%p", ws->e);
+	} else {
+		VSB_printf(pan_vsp, " BAD_MAGIC(0x%08x) ", ws->magic);
+	}
 	VSB_printf(pan_vsp, "},\n");
 	VSB_printf(pan_vsp, "%*s},\n", indent, "" );
 }



From phk at FreeBSD.org  Mon Dec  9 18:51:43 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 09 Dec 2013 19:51:43 +0100
Subject: [master] e09222a Try to unbreak make distcheck
Message-ID: <E1Vq5vj-00041z-CK@project.varnish-software.com>


commit e09222ae1e456a7e71d8bfb71f4856c4e502bcda
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 9 18:51:24 2013 +0000

    Try to unbreak make distcheck

diff --git a/autogen.des b/autogen.des
index d546bc7..a385c71 100755
--- a/autogen.des
+++ b/autogen.des
@@ -18,6 +18,7 @@ rm -f configure
 # autoconf prior to 2.62 has issues with zsh 4.2 and newer
 export CONFIG_SHELL=/bin/sh
 
+env MAKE=gmake \
 ./configure \
     --enable-developer-warnings \
     --enable-debugging-symbols \
diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am
index f73720c..18d5ecf 100644
--- a/lib/libvmod_debug/Makefile.am
+++ b/lib/libvmod_debug/Makefile.am
@@ -29,4 +29,6 @@ vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h
+CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+	$(builddir)/vmod_debug.rst \
+	$(builddir)/vmod_debug.man.rst
diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index 7e2e2f3..3c2c378 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -33,4 +33,6 @@ vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h
+CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+	$(builddir)/vmod_directors.rst \
+	$(builddir)/vmod_directors.man.rst
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index cd55e5b..cd8aebf 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -33,7 +33,9 @@ vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h
+CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+	$(builddir)/vmod_std.rst \
+	$(builddir)/vmod_std.man.rst
 
 vmod_std.3: $(top_srcdir)/doc/sphinx/reference/vmod_std.rst
 if HAVE_RST2MAN



From phk at FreeBSD.org  Mon Dec  9 19:12:10 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 09 Dec 2013 20:12:10 +0100
Subject: [master] 642e23e A debugging  line slipped in
Message-ID: <E1Vq6FW-000649-Vp@project.varnish-software.com>


commit 642e23ea515a1791e09ddd2441a5b11b75917043
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 9 19:11:57 2013 +0000

    A debugging  line slipped in

diff --git a/autogen.des b/autogen.des
index a385c71..d546bc7 100755
--- a/autogen.des
+++ b/autogen.des
@@ -18,7 +18,6 @@ rm -f configure
 # autoconf prior to 2.62 has issues with zsh 4.2 and newer
 export CONFIG_SHELL=/bin/sh
 
-env MAKE=gmake \
 ./configure \
     --enable-developer-warnings \
     --enable-debugging-symbols \



From phk at FreeBSD.org  Tue Dec 10 09:53:32 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Dec 2013 10:53:32 +0100
Subject: [master] f29735e Improve descriptions
Message-ID: <E1VqK0S-00020L-Te@project.varnish-software.com>


commit f29735ee23c79f228811b95d93906122c03c630f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 10 09:53:08 2013 +0000

    Improve descriptions

diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index 09dbe80..6c4e1e1 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -452,7 +452,11 @@ struct parspec mgt_parspec[] = {
 		WAITER_DEFAULT, NULL },
 	{ "ban_dups", tweak_bool, &mgt_param.ban_dups,
 		NULL, NULL,
-		"Detect and eliminate duplicate bans.",
+		"Elimited older identical bans when new bans are created."
+		"  This test is CPU intensive and scales with the number and"
+		" complexity of active (non-Gone) bans.  If identical bans"
+		" are frequent, the amount of CPU needed to actually test "
+		" the bans will be similarly reduced.",
 		0,
 		"on", "bool" },
 	{ "syslog_cli_traffic", tweak_bool, &mgt_param.syslog_cli_traffic,
@@ -463,9 +467,10 @@ struct parspec mgt_parspec[] = {
 	{ "ban_lurker_sleep", tweak_timeout,
 		&mgt_param.ban_lurker_sleep,
 		"0", NULL,
-		"How long time does the ban lurker thread sleeps between "
-		"successful attempts to push the last item up the ban "
-		" list.  It always sleeps a second when nothing can be done.\n"
+		"The ban lurker thread sleeps between work batches, in order"
+		" to not monopolize CPU power."
+		"  When nothing is done, it sleeps a fraction of a second"
+		" before looking for new work to do.\n"
 		"A value of zero disables the ban lurker.",
 		0,
 		"0.01", "s" },



From phk at FreeBSD.org  Tue Dec 10 12:17:44 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Dec 2013 13:17:44 +0100
Subject: [master] 14b7007 Improve error reporting in VCL::ban() by emitting
 the errors as VCL_Error SLTs.
Message-ID: <E1VqMG0-0007MI-J1@project.varnish-software.com>


commit 14b70079423315b0b61a41910fadb72b861b0a6a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 10 12:16:55 2013 +0000

    Improve error reporting in VCL::ban() by emitting the errors as
    VCL_Error SLTs.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 08f81b2..1cf8018 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -827,10 +827,9 @@ void VBP_Init(void);
 
 /* cache_ban.c */
 struct ban *BAN_New(void);
-int BAN_AddTest(struct cli *, struct ban *, const char *, const char *,
-    const char *);
+int BAN_AddTest(struct ban *, const char *, const char *, const char *);
 void BAN_Free(struct ban *b);
-int BAN_Insert(struct ban *b);
+char *BAN_Insert(struct ban *b);
 void BAN_Init(void);
 void BAN_Shutdown(void);
 void BAN_NewObjCore(struct objcore *oc);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index bb22fa4..a044504 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -64,6 +64,7 @@
 
 #include <math.h>
 #include <pcre.h>
+#include <stdarg.h>
 #include <stdio.h>
 
 #include "cache.h"
@@ -83,6 +84,7 @@ struct ban {
 	int			refcount;
 	unsigned		flags;
 #define BAN_F_GONE		(1 << 0)
+#define BAN_F_ERROR		(1 << 1)	/* sticky error marker */
 #define BAN_F_REQ		(1 << 2)
 #define BAN_F_LURK		(3 << 6)	/* ban-lurker-color */
 	VTAILQ_HEAD(,objcore)	objcore;
@@ -325,6 +327,30 @@ ban_iter(const uint8_t **bs, struct ban_test *bt)
 }
 
 /*--------------------------------------------------------------------
+ */
+
+static int
+ban_error(struct ban *b, const char *fmt, ...)
+{
+	va_list ap;
+
+	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+	AN(b->vsb);
+
+	/* First error is sticky */
+	if (!(b->flags & BAN_F_ERROR)) {
+		b->flags |= BAN_F_ERROR;
+
+		/* Record the error message in the vsb */
+		VSB_clear(b->vsb);
+		va_start(ap, fmt);
+		(void)VSB_vprintf(b->vsb, fmt, ap);
+		va_end(ap);
+	}
+	return (-1);
+}
+
+/*--------------------------------------------------------------------
  * Parse and add a http argument specification
  * Output something which HTTP_GetHdr understands
  */
@@ -347,7 +373,7 @@ ban_parse_http(const struct ban *b, const char *a1)
  */
 
 static int
-ban_parse_regexp(struct cli *cli, const struct ban *b, const char *a3)
+ban_parse_regexp(struct ban *b, const char *a3)
 {
 	const char *error;
 	int erroroffset, rc;
@@ -355,12 +381,8 @@ ban_parse_regexp(struct cli *cli, const struct ban *b, const char *a3)
 	pcre *re;
 
 	re = pcre_compile(a3, 0, &error, &erroroffset, NULL);
-	if (re == NULL) {
-		VSL(SLT_Debug, 0, "REGEX: <%s>", error);
-		VCLI_Out(cli, "%s", error);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return (-1);
-	}
+	if (re == NULL)
+		return (ban_error(b, "Regex compile error: %s", error));
 	rc = pcre_fullinfo(re, NULL, PCRE_INFO_SIZE, &sz);
 	AZ(rc);
 	ban_add_lump(b, re, sz);
@@ -372,22 +394,27 @@ ban_parse_regexp(struct cli *cli, const struct ban *b, const char *a3)
  */
 
 int
-BAN_AddTest(struct cli *cli, struct ban *b, const char *a1, const char *a2,
-    const char *a3)
+BAN_AddTest(struct ban *b, const char *a1, const char *a2, const char *a3)
 {
 	const struct pvar *pv;
 	int i;
 
 	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+	AN(b->vsb);
+	AN(a1);
+	AN(a2);
+	AN(a3);
+
+	if (b->flags & BAN_F_ERROR)
+		return (-1);
 
 	for (pv = pvars; pv->name != NULL; pv++)
 		if (!strncmp(a1, pv->name, strlen(pv->name)))
 			break;
-	if (pv->name == NULL) {
-		VCLI_Out(cli, "unknown or unsupported field \"%s\"", a1);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return (-1);
-	}
+
+	if (pv->name == NULL)
+		return (ban_error(b,
+		    "Unknown or unsupported field \"%s\"", a1));
 
 	if (pv->flag & PVAR_REQ)
 		b->flags |= BAN_F_REQ;
@@ -399,12 +426,12 @@ BAN_AddTest(struct cli *cli, struct ban *b, const char *a1, const char *a2,
 	ban_add_lump(b, a3, strlen(a3) + 1);
 	if (!strcmp(a2, "~")) {
 		VSB_putc(b->vsb, BANS_OPER_MATCH);
-		i = ban_parse_regexp(cli, b, a3);
+		i = ban_parse_regexp(b, a3);
 		if (i)
 			return (i);
 	} else if (!strcmp(a2, "!~")) {
 		VSB_putc(b->vsb, BANS_OPER_NMATCH);
-		i = ban_parse_regexp(cli, b, a3);
+		i = ban_parse_regexp(b, a3);
 		if (i)
 			return (i);
 	} else if (!strcmp(a2, "==")) {
@@ -412,10 +439,8 @@ BAN_AddTest(struct cli *cli, struct ban *b, const char *a1, const char *a2,
 	} else if (!strcmp(a2, "!=")) {
 		VSB_putc(b->vsb, BANS_OPER_NEQ);
 	} else {
-		VCLI_Out(cli,
-		    "expected conditional (~, !~, == or !=) got \"%s\"", a2);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return (-1);
+		return (ban_error(b,
+		    "expected conditional (~, !~, == or !=) got \"%s\"", a2));
 	}
 	return (0);
 }
@@ -432,26 +457,50 @@ BAN_AddTest(struct cli *cli, struct ban *b, const char *a1, const char *a2,
  *      deleted.
  */
 
-int
+static char *
+ban_ins_error(const char *p)
+{
+	char *r = NULL;
+	static char nomem[] = "Could not get memory";
+
+	if (p != NULL)
+		r = strdup(p);
+	if (r == NULL)
+		r = nomem;
+	return (r);
+}
+
+char *
 BAN_Insert(struct ban *b)
 {
 	struct ban  *bi, *be;
 	ssize_t ln;
 	double t0;
+	char *p;
 
 	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+	AN(b->vsb);
 
 	if (ban_shutdown) {
 		BAN_Free(b);
-		return (-1);
+		return (ban_ins_error("Shutting down"));
 	}
 
 	AZ(VSB_finish(b->vsb));
 	ln = VSB_len(b->vsb);
 	assert(ln >= 0);
 
+	if (b->flags & BAN_F_ERROR) {
+		p = ban_ins_error(VSB_data(b->vsb));
+		BAN_Free(b);
+		return (p);
+	}
+
 	b->spec = malloc(ln + BANS_HEAD_LEN);
-	XXXAN(b->spec);
+	if (b->spec == NULL) {
+		BAN_Free(b);
+		return (ban_ins_error(NULL));
+	}
 
 	t0 = VTIM_real();
 	memcpy(b->spec + BANS_TIMESTAMP, &t0, sizeof t0);
@@ -468,7 +517,7 @@ BAN_Insert(struct ban *b)
 		/* Check again, we might have raced */
 		Lck_Unlock(&ban_mtx);
 		BAN_Free(b);
-		return (-1);
+		return (ban_ins_error("Shutting down"));
 	}
 	VTAILQ_INSERT_HEAD(&ban_head, b, list);
 	ban_start = b;
@@ -496,7 +545,7 @@ BAN_Insert(struct ban *b)
 	Lck_Unlock(&ban_mtx);
 
 	if (be == NULL)
-		return (0);
+		return (NULL);
 
 	/* Hunt down duplicates, and mark them as gone */
 	bi = b;
@@ -513,7 +562,7 @@ BAN_Insert(struct ban *b)
 	be->refcount--;
 	Lck_Unlock(&ban_mtx);
 
-	return (0);
+	return (NULL);
 }
 
 /*--------------------------------------------------------------------
@@ -1140,6 +1189,7 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv)
 {
 	int narg, i;
 	struct ban *b;
+	char *p;
 
 	(void)priv;
 
@@ -1166,14 +1216,13 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv)
 		return;
 	}
 	for (i = 0; i < narg; i += 4)
-		if (BAN_AddTest(cli, b, av[i + 2], av[i + 3], av[i + 4])) {
-			BAN_Free(b);
-			return;
-		}
-	if (BAN_Insert(b) < 0) {
-		VCLI_Out(cli, "Shutdown in progress");
-		VCLI_SetResult(cli, CLIS_CANT);
-		return;
+		if (BAN_AddTest(b, av[i + 2], av[i + 3], av[i + 4]))
+			break;
+	p = BAN_Insert(b);
+	if (p != NULL) {
+		VCLI_Out(cli, "%s", p);
+		free(p);
+		VCLI_SetResult(cli, CLIS_PARAM);
 	}
 }
 
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 563b2bc..a1223fd 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -406,46 +406,65 @@ VRT_synth_page(const struct vrt_ctx *ctx, unsigned flags, const char *str, ...)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_ban_string(const char *str)
+VRT_ban_string(const struct vrt_ctx *ctx, const char *str)
 {
 	char *a1, *a2, *a3;
 	char **av;
 	struct ban *b;
-	int good;
 	int i;
 
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(ctx->vsl);
+	AN(str);
+
+	b = BAN_New();
+	if (b == NULL) {
+		VSLb(ctx->vsl, SLT_VCL_Error, "ban(): Out of Memory");
+		return;
+	}
 	av = VAV_Parse(str, NULL, ARGV_NOESC);
+	AN(av);
 	if (av[0] != NULL) {
-		/* XXX: report error how ? */
+		VSLb(ctx->vsl, SLT_VCL_Error, "ban(): %s", av[0]);
 		VAV_Free(av);
+		BAN_Free(b);
 		return;
 	}
-	b = BAN_New();
-	good = 0;
-	for (i = 1; ;) {
-		a1 = av[i++];
-		if (a1 == NULL)
+	for (i = 0; ;) {
+		a1 = av[++i];
+		if (a1 == NULL) {
+			VSLb(ctx->vsl, SLT_VCL_Error,
+			    "ban(): No ban conditions found.");
 			break;
-		good = 0;
-		a2 = av[i++];
-		if (a2 == NULL)
-			break;
-		a3 = av[i++];
-		if (a3 == NULL)
+		}
+		a2 = av[++i];
+		if (a2 == NULL) {
+			VSLb(ctx->vsl, SLT_VCL_Error,
+			    "ban(): Expected comparison operator.");
 			break;
-		if (BAN_AddTest(NULL, b, a1, a2, a3))
+		}
+		a3 = av[++i];
+		if (a3 == NULL) {
+			VSLb(ctx->vsl, SLT_VCL_Error,
+			    "ban(): Expected second operand.");
 			break;
-		good = 1;
-		if (av[i] == NULL)
+		}
+		if (BAN_AddTest(b, a1, a2, a3) || av[++i] == NULL) {
+			a1 = BAN_Insert(b);
+			if (a1 != NULL) {
+				VSLb(ctx->vsl, SLT_VCL_Error,
+				    "ban(): %s", a1);
+				free(a1);
+			}
 			break;
-		good = 0;
-		if (strcmp(av[i++], "&&"))
+		}
+		if (strcmp(av[i], "&&")) {
+			VSLb(ctx->vsl, SLT_VCL_Error,
+			    "ban(): Expected && between conditions,"
+			    " found \"%s\"", av[i]);
 			break;
+		}
 	}
-	if (!good)
-		BAN_Free(b);		/* XXX: report error how ? */
-	else
-		(void)BAN_Insert(b);	/* XXX: report error how ? */
 	VAV_Free(av);
 }
 
diff --git a/bin/varnishtest/tests/v00011.vtc b/bin/varnishtest/tests/v00011.vtc
index d9fad31..162c66a 100644
--- a/bin/varnishtest/tests/v00011.vtc
+++ b/bin/varnishtest/tests/v00011.vtc
@@ -11,6 +11,12 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.method == "PURGE") {
+			ban("");
+			ban("req.url");
+			ban("req.url //");
+			ban("req.url // bar");
+			ban("req.url == bar //");
+			ban("foo == bar //");
 			ban("req.url ~ ^/$");
 			return (error(209,"foo"));
 		}
@@ -24,6 +30,16 @@ client c1 {
 	expect resp.http.X-Varnish == "1001"
 } -run
 
+logexpect l1 -v v1 -d 1 -g vxid {
+	expect * 1004	VCL_Error {ban[(][)]: No ban conditions found[.]}
+	expect * 1004	VCL_Error {ban[(][)]: Expected comparison operator[.]}
+	expect * 1004	VCL_Error {ban[(][)]: Expected second operand[.]}
+	expect * 1004	VCL_Error {ban[(][)]: expected conditional [(]~, !~, == or !=[)] got "//"}
+	expect * 1004	VCL_Error {ban[(][)]: Expected && between conditions, found .//.}
+	expect * 1004	VCL_Error {ban[(][)]: Unknown or unsupported field .foo.}
+
+} -start
+
 client c1 {
 	txreq -req "PURGE"
 	rxresp
@@ -31,6 +47,8 @@ client c1 {
 	expect resp.status == 209
 } -run
 
+logexpect l1 -wait
+
 client c1 {
 	txreq
 	rxresp
diff --git a/include/vrt.h b/include/vrt.h
index fa512dd..6dfefee 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -199,7 +199,7 @@ int VRT_re_match(const struct vrt_ctx *, const char *, void *re);
 const char *VRT_regsub(const struct vrt_ctx *, int all, const char *,
     void *, const char *);
 
-void VRT_ban_string(const char *);
+void VRT_ban_string(const struct vrt_ctx *, const char *);
 void VRT_purge(const struct vrt_ctx *, double ttl, double grace);
 
 void VRT_count(const struct vrt_ctx *, unsigned);
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 4e441d2..79786ef 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -263,7 +263,7 @@ parse_ban(struct vcc *tl)
 	ExpectErr(tl, '(');
 	vcc_NextToken(tl);
 
-	Fb(tl, 1, "VRT_ban_string(\n");
+	Fb(tl, 1, "VRT_ban_string(ctx, \n");
 	tl->indent += INDENT;
 	vcc_Expr(tl, STRING);
 	tl->indent -= INDENT;



From phk at FreeBSD.org  Tue Dec 10 19:38:59 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Dec 2013 20:38:59 +0100
Subject: [master] e01e604 Mark bans both with a "uses req.*" and a "uses
 obj.*" flag,
 the lurker can actually make useful determinations on "mixed" bans.
Message-ID: <E1VqT91-0002DH-7w@project.varnish-software.com>


commit e01e6042f59c2f72b256a393308b03d45ef46b57
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 10 19:38:12 2013 +0000

    Mark bans both with a "uses req.*" and a "uses obj.*" flag, the
    lurker can actually make useful determinations on "mixed" bans.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index a044504..f50b1b5 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -86,12 +86,15 @@ struct ban {
 #define BAN_F_GONE		(1 << 0)
 #define BAN_F_ERROR		(1 << 1)	/* sticky error marker */
 #define BAN_F_REQ		(1 << 2)
+#define BAN_F_OBJ		(1 << 3)
 #define BAN_F_LURK		(3 << 6)	/* ban-lurker-color */
 	VTAILQ_HEAD(,objcore)	objcore;
 	struct vsb		*vsb;
 	uint8_t			*spec;
 };
 
+VTAILQ_HEAD(banhead_s,ban);
+
 #define LURK_SHIFT 6
 
 struct ban_test {
@@ -103,7 +106,7 @@ struct ban_test {
 };
 
 static void ban_info(enum baninfo event, const uint8_t *ban, unsigned len);
-static VTAILQ_HEAD(banhead_s,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
+static struct banhead_s ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
 static struct lock ban_mtx;
 static struct ban *ban_magic;
 static pthread_t ban_thread;
@@ -119,8 +122,10 @@ static int ban_shutdown = 0;
 #define BANS_LENGTH		8
 #define BANS_FLAGS		12
 #define BANS_HEAD_LEN		13
+
 #define BANS_FLAG_REQ		0x01
-#define BANS_FLAG_GONE		0x02
+#define BANS_FLAG_OBJ		0x02
+#define BANS_FLAG_GONE		0x04
 
 #define BANS_OPER_EQ		0x10
 #define BANS_OPER_NEQ		0x11
@@ -418,6 +423,8 @@ BAN_AddTest(struct ban *b, const char *a1, const char *a2, const char *a3)
 
 	if (pv->flag & PVAR_REQ)
 		b->flags |= BAN_F_REQ;
+	if (pv->flag & PVAR_OBJ)
+		b->flags |= BAN_F_OBJ;
 
 	VSB_putc(b->vsb, pv->tag);
 	if (pv->flag & PVAR_HTTP)
@@ -504,7 +511,11 @@ BAN_Insert(struct ban *b)
 
 	t0 = VTIM_real();
 	memcpy(b->spec + BANS_TIMESTAMP, &t0, sizeof t0);
-	b->spec[BANS_FLAGS] = (b->flags & BAN_F_REQ) ? BANS_FLAG_REQ : 0;
+	b->spec[BANS_FLAGS] = 0;
+	if (b->flags & BAN_F_REQ)
+		b->spec[BANS_FLAGS] |= BANS_FLAG_REQ;
+	if (b->flags & BAN_F_OBJ)
+		b->spec[BANS_FLAGS] |= BANS_FLAG_OBJ;
 	memcpy(b->spec + BANS_HEAD_LEN, VSB_data(b->vsb), ln);
 	ln += BANS_HEAD_LEN;
 	vbe32enc(b->spec + BANS_LENGTH, ln);
diff --git a/include/tbl/ban_vars.h b/include/tbl/ban_vars.h
index abf083e..0926fe6 100644
--- a/include/tbl/ban_vars.h
+++ b/include/tbl/ban_vars.h
@@ -29,10 +29,11 @@
  *
  */
 
-#define PVAR_HTTP	1
-#define PVAR_REQ	2
+#define PVAR_HTTP	(1<<0)
+#define PVAR_REQ	(1<<1)
+#define PVAR_OBJ	(1<<2)
 
 PVAR("req.url",		PVAR_REQ,		BANS_ARG_URL)
-PVAR("req.http.",	PVAR_REQ|PVAR_HTTP,	BANS_ARG_REQHTTP)
-PVAR("obj.http.",	PVAR_HTTP,		BANS_ARG_OBJHTTP)
-PVAR("obj.status",	0,			BANS_ARG_OBJSTATUS)
+PVAR("req.http.",	PVAR_REQ | PVAR_HTTP,	BANS_ARG_REQHTTP)
+PVAR("obj.status",	PVAR_OBJ,		BANS_ARG_OBJSTATUS)
+PVAR("obj.http.",	PVAR_OBJ | PVAR_HTTP,	BANS_ARG_OBJHTTP)



From phk at FreeBSD.org  Tue Dec 10 23:02:52 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 11 Dec 2013 00:02:52 +0100
Subject: [master] e82fec9 Sanitize the three different bitmaps of flags in BAN
 into single one.
Message-ID: <E1VqWKK-0000LN-E8@project.varnish-software.com>


commit e82fec98af207fe4d564f6608a1b8decc8fe6b26
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 10 23:02:16 2013 +0000

    Sanitize the three different bitmaps of flags in BAN into single one.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index f50b1b5..ec309af 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -41,7 +41,7 @@
  * Bans are compiled into bytestrings as follows:
  *	8 bytes	- double: timestamp		XXX: Byteorder ?
  *	4 bytes - be32: length
- *	1 byte - flags: 0x01: BAN_F_REQ
+ *	1 byte - flags: 0x01: BAN_F_{REQ|OBJ|GONE}
  *	N tests
  * A test have this form:
  *	1 byte - arg (see ban_vars.h col 3 "BANS_ARG_XXX")
@@ -77,16 +77,40 @@
 #include "vmb.h"
 #include "vtim.h"
 
+/*--------------------------------------------------------------------
+ * BAN string defines & magic markers
+ */
+
+#define BANS_TIMESTAMP		0
+#define BANS_LENGTH		8
+#define BANS_FLAGS		12
+#define BANS_HEAD_LEN		13
+
+#define BANS_FLAG_REQ		(1<<0)
+#define BANS_FLAG_OBJ		(1<<1)
+#define BANS_FLAG_GONE		(1<<2)
+#define BANS_FLAG_HTTP		(1<<3)
+#define BANS_FLAG_ERROR		(1<<4)
+
+#define BANS_OPER_EQ		0x10
+#define BANS_OPER_NEQ		0x11
+#define BANS_OPER_MATCH		0x12
+#define BANS_OPER_NMATCH	0x13
+
+#define BANS_ARG_URL		0x18
+#define BANS_ARG_REQHTTP	0x19
+#define BANS_ARG_OBJHTTP	0x1a
+#define BANS_ARG_OBJSTATUS	0x1b
+
+/*--------------------------------------------------------------------*/
+
 struct ban {
 	unsigned		magic;
 #define BAN_MAGIC		0x700b08ea
 	VTAILQ_ENTRY(ban)	list;
 	int			refcount;
-	unsigned		flags;
-#define BAN_F_GONE		(1 << 0)
-#define BAN_F_ERROR		(1 << 1)	/* sticky error marker */
-#define BAN_F_REQ		(1 << 2)
-#define BAN_F_OBJ		(1 << 3)
+	unsigned		flags;		/* BANS_FLAG_* */
+
 #define BAN_F_LURK		(3 << 6)	/* ban-lurker-color */
 	VTAILQ_HEAD(,objcore)	objcore;
 	struct vsb		*vsb;
@@ -115,29 +139,6 @@ static bgthread_t ban_lurker;
 static int ban_shutdown = 0;
 
 /*--------------------------------------------------------------------
- * BAN string defines & magic markers
- */
-
-#define BANS_TIMESTAMP		0
-#define BANS_LENGTH		8
-#define BANS_FLAGS		12
-#define BANS_HEAD_LEN		13
-
-#define BANS_FLAG_REQ		0x01
-#define BANS_FLAG_OBJ		0x02
-#define BANS_FLAG_GONE		0x04
-
-#define BANS_OPER_EQ		0x10
-#define BANS_OPER_NEQ		0x11
-#define BANS_OPER_MATCH		0x12
-#define BANS_OPER_NMATCH	0x13
-
-#define BANS_ARG_URL		0x18
-#define BANS_ARG_REQHTTP	0x19
-#define BANS_ARG_OBJHTTP	0x1a
-#define BANS_ARG_OBJSTATUS	0x1b
-
-/*--------------------------------------------------------------------
  * Variables we can purge on
  */
 
@@ -274,9 +275,9 @@ ban_mark_gone(struct ban *b)
 
 	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
 	AN(b->spec);
-	AZ(b->flags & BAN_F_GONE);
+	AZ(b->flags & BANS_FLAG_GONE);
 	ln = ban_len(b->spec);
-	b->flags |= BAN_F_GONE;
+	b->flags |= BANS_FLAG_GONE;
 	b->spec[BANS_FLAGS] |= BANS_FLAG_GONE;
 	VWMB();
 	vbe32enc(b->spec + BANS_LENGTH, BANS_HEAD_LEN);
@@ -343,8 +344,8 @@ ban_error(struct ban *b, const char *fmt, ...)
 	AN(b->vsb);
 
 	/* First error is sticky */
-	if (!(b->flags & BAN_F_ERROR)) {
-		b->flags |= BAN_F_ERROR;
+	if (!(b->flags & BANS_FLAG_ERROR)) {
+		b->flags |= BANS_FLAG_ERROR;
 
 		/* Record the error message in the vsb */
 		VSB_clear(b->vsb);
@@ -410,7 +411,7 @@ BAN_AddTest(struct ban *b, const char *a1, const char *a2, const char *a3)
 	AN(a2);
 	AN(a3);
 
-	if (b->flags & BAN_F_ERROR)
+	if (b->flags & BANS_FLAG_ERROR)
 		return (-1);
 
 	for (pv = pvars; pv->name != NULL; pv++)
@@ -421,13 +422,10 @@ BAN_AddTest(struct ban *b, const char *a1, const char *a2, const char *a3)
 		return (ban_error(b,
 		    "Unknown or unsupported field \"%s\"", a1));
 
-	if (pv->flag & PVAR_REQ)
-		b->flags |= BAN_F_REQ;
-	if (pv->flag & PVAR_OBJ)
-		b->flags |= BAN_F_OBJ;
+	b->flags |= pv->flag;
 
 	VSB_putc(b->vsb, pv->tag);
-	if (pv->flag & PVAR_HTTP)
+	if (pv->flag & BANS_FLAG_HTTP)
 		ban_parse_http(b, a1 + strlen(pv->name));
 
 	ban_add_lump(b, a3, strlen(a3) + 1);
@@ -497,7 +495,7 @@ BAN_Insert(struct ban *b)
 	ln = VSB_len(b->vsb);
 	assert(ln >= 0);
 
-	if (b->flags & BAN_F_ERROR) {
+	if (b->flags & BANS_FLAG_ERROR) {
 		p = ban_ins_error(VSB_data(b->vsb));
 		BAN_Free(b);
 		return (p);
@@ -511,11 +509,7 @@ BAN_Insert(struct ban *b)
 
 	t0 = VTIM_real();
 	memcpy(b->spec + BANS_TIMESTAMP, &t0, sizeof t0);
-	b->spec[BANS_FLAGS] = 0;
-	if (b->flags & BAN_F_REQ)
-		b->spec[BANS_FLAGS] |= BANS_FLAG_REQ;
-	if (b->flags & BAN_F_OBJ)
-		b->spec[BANS_FLAGS] |= BANS_FLAG_OBJ;
+	b->spec[BANS_FLAGS] = b->flags & 0xff;
 	memcpy(b->spec + BANS_HEAD_LEN, VSB_data(b->vsb), ln);
 	ln += BANS_HEAD_LEN;
 	vbe32enc(b->spec + BANS_LENGTH, ln);
@@ -534,7 +528,7 @@ BAN_Insert(struct ban *b)
 	ban_start = b;
 	VSC_C_main->bans++;
 	VSC_C_main->bans_added++;
-	if (b->flags & BAN_F_REQ)
+	if (b->flags & BANS_FLAG_REQ)
 		VSC_C_main->bans_req++;
 
 	be = VTAILQ_LAST(&ban_head, banhead_s);
@@ -563,7 +557,7 @@ BAN_Insert(struct ban *b)
 	Lck_Lock(&ban_mtx);
 	while(!ban_shutdown && bi != be) {
 		bi = VTAILQ_NEXT(bi, list);
-		if (bi->flags & BAN_F_GONE)
+		if (bi->flags & BANS_FLAG_GONE)
 			continue;
 		if (!ban_equal(b->spec, bi->spec))
 			continue;
@@ -726,7 +720,7 @@ ban_reload(const uint8_t *ban, unsigned len)
 	memcpy(b2->spec, ban, len);
 	if (ban[BANS_FLAGS] & BANS_FLAG_REQ) {
 		VSC_C_main->bans_req++;
-		b2->flags |= BAN_F_REQ;
+		b2->flags |= BANS_FLAG_REQ;
 	}
 	if (duplicate)
 		VSC_C_main->bans_dups++;
@@ -740,7 +734,7 @@ ban_reload(const uint8_t *ban, unsigned len)
 
 	/* Hunt down older duplicates */
 	for (b = VTAILQ_NEXT(b2, list); b != NULL; b = VTAILQ_NEXT(b, list)) {
-		if (b->flags & BAN_F_GONE)
+		if (b->flags & BANS_FLAG_GONE)
 			continue;
 		if (ban_equal(b->spec, ban)) {
 			ban_mark_gone(b);
@@ -922,15 +916,15 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	skipped = 0;
 	for (b = b0; b != oc->ban; b = VTAILQ_NEXT(b, list)) {
 		CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
-		if (b->flags & BAN_F_GONE)
+		if (b->flags & BANS_FLAG_GONE)
 			continue;
 		if ((b->flags & BAN_F_LURK) &&
 		    (b->flags & BAN_F_LURK) == (oc->flags & OC_F_LURK)) {
-			AZ(b->flags & BAN_F_REQ);
+			AZ(b->flags & BANS_FLAG_REQ);
 			/* Lurker already tested this */
 			continue;
 		}
-		if (req_http == NULL && (b->flags & BAN_F_REQ)) {
+		if (req_http == NULL && (b->flags & BANS_FLAG_REQ)) {
 			/*
 			 * We cannot test this one, but there might
 			 * be other bans that match, so we soldier on
@@ -991,9 +985,9 @@ ban_cleantail(void)
 		Lck_Lock(&ban_mtx);
 		b = VTAILQ_LAST(&ban_head, banhead_s);
 		if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
-			if (b->flags & BAN_F_GONE)
+			if (b->flags & BANS_FLAG_GONE)
 				VSC_C_main->bans_gone--;
-			if (b->flags & BAN_F_REQ)
+			if (b->flags & BANS_FLAG_REQ)
 				VSC_C_main->bans_req--;
 			VSC_C_main->bans--;
 			VSC_C_main->bans_deleted++;
@@ -1034,9 +1028,9 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 	i = 0;
 	b0 = NULL;
 	VTAILQ_FOREACH(b, &ban_head, list) {
-		if (b->flags & BAN_F_GONE)
+		if (b->flags & BANS_FLAG_GONE)
 			continue;
-		if (b->flags & BAN_F_REQ)
+		if (b->flags & BANS_FLAG_REQ)
 			continue;
 		if (b == VTAILQ_LAST(&ban_head, banhead_s))
 			continue;
@@ -1134,8 +1128,8 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 			VTIM_sleep(cache_param->ban_lurker_sleep);
 		}
 		Lck_AssertHeld(&ban_mtx);
-		if (!(b->flags & BAN_F_REQ)) {
-			if (!(b->flags & BAN_F_GONE))
+		if (!(b->flags & BANS_FLAG_REQ)) {
+			if (!(b->flags & BANS_FLAG_GONE))
 				ban_mark_gone(b);
 			if (DO_DEBUG(DBG_LURKER))
 				VSLb(vsl, SLT_Debug, "lurker BAN %f now gone",
@@ -1294,7 +1288,7 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 	VTAILQ_FOREACH(b, &ban_head, list) {
 		VCLI_Out(cli, "%10.6f %5u%s\t", ban_time(b->spec),
 		    bl == b ? b->refcount - 1 : b->refcount,
-		    b->flags & BAN_F_GONE ? "G" : " ");
+		    b->flags & BANS_FLAG_GONE ? "G" : " ");
 		ban_render(cli, b->spec);
 		VCLI_Out(cli, "\n");
 		if (VCLI_Overflow(cli))
diff --git a/include/tbl/ban_vars.h b/include/tbl/ban_vars.h
index 0926fe6..a3df78b 100644
--- a/include/tbl/ban_vars.h
+++ b/include/tbl/ban_vars.h
@@ -29,11 +29,7 @@
  *
  */
 
-#define PVAR_HTTP	(1<<0)
-#define PVAR_REQ	(1<<1)
-#define PVAR_OBJ	(1<<2)
-
-PVAR("req.url",		PVAR_REQ,		BANS_ARG_URL)
-PVAR("req.http.",	PVAR_REQ | PVAR_HTTP,	BANS_ARG_REQHTTP)
-PVAR("obj.status",	PVAR_OBJ,		BANS_ARG_OBJSTATUS)
-PVAR("obj.http.",	PVAR_OBJ | PVAR_HTTP,	BANS_ARG_OBJHTTP)
+PVAR("req.url",		BANS_FLAG_REQ,			BANS_ARG_URL)
+PVAR("req.http.",	BANS_FLAG_REQ | BANS_FLAG_HTTP,	BANS_ARG_REQHTTP)
+PVAR("obj.status",	BANS_FLAG_OBJ,			BANS_ARG_OBJSTATUS)
+PVAR("obj.http.",	BANS_FLAG_OBJ | BANS_FLAG_HTTP,	BANS_ARG_OBJHTTP)



From phk at FreeBSD.org  Wed Dec 11 10:11:09 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 11 Dec 2013 11:11:09 +0100
Subject: [master] 119c610 Add a couple of protective asserts
Message-ID: <E1Vqgl3-0000uW-8H@project.varnish-software.com>


commit 119c61073bc3eb097094653a0766ef200fdc6c09
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Dec 11 10:10:53 2013 +0000

    Add a couple of protective asserts

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 1e735b2..5519168 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -221,6 +221,7 @@ VBO_extend(struct busyobj *bo, ssize_t l)
 	assert(l > 0);
 	Lck_Lock(&bo->mtx);
 	st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
+	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 	st->len += l;
 	bo->fetch_obj->len += l;
 	AZ(pthread_cond_signal(&bo->cond));
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 5b90f94..68dbe61 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -768,6 +768,7 @@ HSH_DerefObjCore(struct dstat *ds, struct objcore **ocp)
 	unsigned r;
 
 	AN(ocp);
+	AN(ds);
 	oc = *ocp;
 	*ocp = NULL;
 



From tfheen at varnish-software.com  Thu Dec 12 07:22:47 2013
From: tfheen at varnish-software.com (Tollef Fog Heen)
Date: Thu, 12 Dec 2013 08:22:47 +0100
Subject: [master] 352aea7 Install vmod_director
Message-ID: <E1Vr0bf-0003L6-Qk@project.varnish-software.com>


commit 352aea73426cbbc94bb60684f4eaa93317681d5f
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Dec 12 08:22:33 2013 +0100

    Install vmod_director
    
    Fixes: #1381

diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index 3c2c378..5706ab8 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -9,7 +9,7 @@ AM_CPPFLAGS = \
 vmoddir = $(pkglibdir)/vmods
 vmod_srcdir = $(top_srcdir)/lib/libvmod_directors
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
-noinst_LTLIBRARIES = libvmod_directors.la
+vmod_LTLIBRARIES = libvmod_directors.la
 
 libvmod_directors_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared -rpath /nowhere
 



From martin at varnish-software.com  Thu Dec 12 12:40:02 2013
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Thu, 12 Dec 2013 13:40:02 +0100
Subject: [master] 061bc5c Add a doc paragraph on how and from what varnishncsa
 produces it's output.
Message-ID: <E1Vr5Yg-0001XL-Uu@project.varnish-software.com>


commit 061bc5ca269c42dab0c5cbe7ca2f4eae69db2382
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Dec 12 13:36:22 2013 +0100

    Add a doc paragraph on how and from what varnishncsa produces it's
    output.

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 13b9c10..3c7aa12 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -25,6 +25,12 @@ DESCRIPTION
 The varnishncsa utility reads varnishd(1) shared memory logs and
 presents them in the Apache / NCSA "combined" log format.
 
+Each log line produced is based on a single Request type transaction
+gathered from the shared memory log. The Request transaction is then
+scanned for the relevant parts in order to output one log line. To
+filter the log lines produced, use the query language to select the
+applicable transactions. Non-request transactions are ignored.
+
 The following options are available:
 
 .. include:: ../../../bin/varnishncsa/varnishncsa_options.rst



From phk at FreeBSD.org  Thu Dec 12 13:17:00 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 12 Dec 2013 14:17:00 +0100
Subject: [master] e7170a0 Rewrite the ban-lurker in light of many changes and
 evidence that it basically didn't work as intended any more.
Message-ID: <E1Vr68S-0000CB-Oq@project.varnish-software.com>


commit e7170a0a8abf07a4251c8849b7aae53d227b4803
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 12 13:16:16 2013 +0000

    Rewrite the ban-lurker in light of many changes and evidence that
    it basically didn't work as intended any more.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 1cf8018..38ca5e3 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -437,7 +437,6 @@ struct objcore {
 #define OC_F_PASS		(1<<2)
 #define OC_F_OFFLRU		(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
-#define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
 #define OC_F_DYING		(1<<7)
 #define OC_F_PRIVATE		(1<<8)
 #define OC_F_FAILED		(1<<9)
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index ec309af..1edad6e 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -108,10 +108,10 @@ struct ban {
 	unsigned		magic;
 #define BAN_MAGIC		0x700b08ea
 	VTAILQ_ENTRY(ban)	list;
+	VTAILQ_ENTRY(ban)	l_list;
 	int			refcount;
 	unsigned		flags;		/* BANS_FLAG_* */
 
-#define BAN_F_LURK		(3 << 6)	/* ban-lurker-color */
 	VTAILQ_HEAD(,objcore)	objcore;
 	struct vsb		*vsb;
 	uint8_t			*spec;
@@ -119,8 +119,6 @@ struct ban {
 
 VTAILQ_HEAD(banhead_s,ban);
 
-#define LURK_SHIFT 6
-
 struct ban_test {
 	uint8_t			arg1;
 	const char		*arg1_spec;
@@ -135,6 +133,7 @@ static struct lock ban_mtx;
 static struct ban *ban_magic;
 static pthread_t ban_thread;
 static struct ban * volatile ban_start;
+static struct objcore oc_marker = { .magic = OBJCORE_MAGIC, };
 static bgthread_t ban_lurker;
 static int ban_shutdown = 0;
 
@@ -274,6 +273,8 @@ ban_mark_gone(struct ban *b)
 	unsigned ln;
 
 	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+	Lck_AssertHeld(&ban_mtx);
+
 	AN(b->spec);
 	AZ(b->flags & BANS_FLAG_GONE);
 	ln = ban_len(b->spec);
@@ -528,6 +529,8 @@ BAN_Insert(struct ban *b)
 	ban_start = b;
 	VSC_C_main->bans++;
 	VSC_C_main->bans_added++;
+	if (b->flags & BANS_FLAG_OBJ)
+		VSC_C_main->bans_obj++;
 	if (b->flags & BANS_FLAG_REQ)
 		VSC_C_main->bans_req++;
 
@@ -901,12 +904,21 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(oc->ban, BAN_MAGIC);
 
+	/* First do an optimistic unlocked check */
 	b0 = ban_start;
 	CHECK_OBJ_NOTNULL(b0, BAN_MAGIC);
 
 	if (b0 == oc->ban)
 		return (0);
 
+	/* If that fails, make a safe check */
+	Lck_Lock(&ban_mtx);
+	b0 = ban_start;
+	Lck_Unlock(&ban_mtx);
+
+	if (b0 == oc->ban)
+		return (0);
+
 	/*
 	 * This loop is safe without locks, because we know we hold
 	 * a refcount on a ban somewhere in the list and we do not
@@ -918,12 +930,6 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 		CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
 		if (b->flags & BANS_FLAG_GONE)
 			continue;
-		if ((b->flags & BAN_F_LURK) &&
-		    (b->flags & BAN_F_LURK) == (oc->flags & OC_F_LURK)) {
-			AZ(b->flags & BANS_FLAG_REQ);
-			/* Lurker already tested this */
-			continue;
-		}
 		if (req_http == NULL && (b->flags & BANS_FLAG_REQ)) {
 			/*
 			 * We cannot test this one, but there might
@@ -951,10 +957,10 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	oc->ban->refcount--;
 	VTAILQ_REMOVE(&oc->ban->objcore, oc, ban_list);
 	if (b == oc->ban) {	/* not banned */
-		b->flags &= ~BAN_F_LURK;
 		VTAILQ_INSERT_TAIL(&b0->objcore, oc, ban_list);
 		b0->refcount++;
 	}
+
 	Lck_Unlock(&ban_mtx);
 
 	if (b == oc->ban) {	/* not banned */
@@ -963,7 +969,8 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 		return (0);
 	} else {
 		oc->ban = NULL;
-		VSLb(vsl, SLT_ExpBan, "%u was banned", o->vxid);
+		VSLb(vsl, SLT_ExpBan, "%u banned lookup", o->vxid);
+		VSC_C_main->bans_obj_killed++;
 		EXP_Rearm(o, o->exp.t_origin, 0, 0, 0);	// XXX fake now
 		return (1);
 	}
@@ -987,6 +994,8 @@ ban_cleantail(void)
 		if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
 			if (b->flags & BANS_FLAG_GONE)
 				VSC_C_main->bans_gone--;
+			if (b->flags & BANS_FLAG_OBJ)
+				VSC_C_main->bans_obj--;
 			if (b->flags & BANS_FLAG_REQ)
 				VSC_C_main->bans_req--;
 			VSC_C_main->bans--;
@@ -1005,147 +1014,155 @@ ban_cleantail(void)
 }
 
 /*--------------------------------------------------------------------
- * Ban lurker thread
+ * Our task here is somewhat tricky:  The canonical locking order is
+ * objhead->mtx first, then ban_mtx, because that is the order which
+ * makes most sense in HSH_Lookup(), but we come the other way.
+ * We optimistically try to get them the other way, and get out of
+ * the way if that fails, and retry again later.
  */
 
-static int
-ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
+static struct objcore *
+ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt)
 {
-	struct ban *b, *b0;
 	struct objhead *oh;
-	struct objcore *oc, *oc2;
+	struct objcore *oc;
+
+	while (1) {
+		Lck_Lock(&ban_mtx);
+		oc = VTAILQ_FIRST(&bt->objcore);
+		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+		if (oc == &oc_marker) {
+			VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
+			Lck_Unlock(&ban_mtx);
+			return (NULL);
+		}
+		oh = oc->objhead;
+		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+		if (!Lck_Trylock(&oh->mtx) && oc->refcnt > 0) {
+			/*
+			 * We got the lock, and the oc is not being
+			 * dismantled under our feet, run with it...
+			 */
+			oc->refcnt += 1;
+			VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
+			VTAILQ_INSERT_TAIL(&bt->objcore, oc, ban_list);
+			Lck_Unlock(&oh->mtx);
+			Lck_Unlock(&ban_mtx);
+			break;
+		}
+
+		/* Try again, later */
+		Lck_Unlock(&ban_mtx);
+		VSC_C_main->bans_lurker_contention++;
+		VSL_Flush(vsl, 0);
+		VTIM_sleep(cache_param->ban_lurker_sleep);
+	}
+	return (oc);
+}
+
+static void
+ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
+    const struct banhead_s *obans)
+{
+	struct ban *bl;
+	struct objcore *oc;
 	struct object *o;
-	static unsigned pass = 1 << LURK_SHIFT;
+	unsigned tests;
 	int i;
 
-	AN(pass & BAN_F_LURK);
-	AZ(pass & ~BAN_F_LURK);
+	(void)wrk;
+	/*
+	 * First see if there is anything to do, and if so, insert marker
+	 */
+	Lck_Lock(&ban_mtx);
+	oc = VTAILQ_FIRST(&bt->objcore);
+	if (oc != NULL)
+		VTAILQ_INSERT_TAIL(&bt->objcore, &oc_marker, ban_list);
+	Lck_Unlock(&ban_mtx);
+	if (oc == NULL)
+		return;
+
+	while (1) {
+		oc = ban_lurker_getfirst(vsl, bt);
+		if (oc == NULL)
+			return;
+		o = oc_getobj(&wrk->stats, oc);
+		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+		i = 0;
+		VTAILQ_FOREACH_REVERSE(bl, obans, banhead_s, l_list) {
+			tests = 0;
+			i = ban_evaluate(bl->spec, o->http, NULL, &tests);
+			VSC_C_main->bans_lurker_tested++;
+			VSC_C_main->bans_lurker_tests_tested += tests;
+			if (i)
+				break;
+		}
+		if (i) {
+			VSLb(vsl, SLT_ExpBan, "%u banned by lurker", o->vxid);
+			EXP_Rearm(o, o->exp.t_origin, 0, 0, 0);	// XXX fake now
+			VSC_C_main->bans_lurker_obj_killed++;
+		}
+		(void)HSH_DerefObjCore(&wrk->stats, &oc);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Ban lurker thread
+ */
+
+static int
+ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
+{
+	struct ban *b, *bt;
+	//struct objhead *oh;
+	// struct objcore *oc, *oc2;
+	//struct object *o;
+	struct banhead_s obans;
+	int i;
 
 	/*
-	 * Find out if we have any bans we can do something about
-	 * If we find any, tag them with our pass number.
+	 * Make a list of the bans we can do something about
 	 */
+
+	VTAILQ_INIT(&obans);
+	Lck_Lock(&ban_mtx);
+	b = ban_start;
+	Lck_Unlock(&ban_mtx);
 	i = 0;
-	b0 = NULL;
-	VTAILQ_FOREACH(b, &ban_head, list) {
-		if (b->flags & BANS_FLAG_GONE)
-			continue;
-		if (b->flags & BANS_FLAG_REQ)
-			continue;
-		if (b == VTAILQ_LAST(&ban_head, banhead_s))
-			continue;
-		if (b0 == NULL)
-			b0 = b;
-		i++;
-		b->flags &= ~BAN_F_LURK;
-		b->flags |= pass;
+	while (b != NULL) {
+		if (b->flags & BANS_FLAG_GONE) {
+			;
+		} else if (b->flags & BANS_FLAG_REQ) {
+			;
+		} else if (b == VTAILQ_LAST(&ban_head, banhead_s)) {
+			;
+		} else {
+			VTAILQ_INSERT_TAIL(&obans, b, l_list);
+			i++;
+		}
+		b = VTAILQ_NEXT(b, list);
 	}
 	if (DO_DEBUG(DBG_LURKER))
 		VSLb(vsl, SLT_Debug, "lurker: %d actionable bans", i);
 	if (i == 0)
 		return (0);
 
-	VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) {
-		if (DO_DEBUG(DBG_LURKER))
-			VSLb(vsl, SLT_Debug, "lurker doing %f %d",
-			    ban_time(b->spec), b->refcount);
-		while (1) {
-			Lck_Lock(&ban_mtx);
-			if (ban_shutdown)
-				break;
-			oc = VTAILQ_FIRST(&b->objcore);
-			if (oc == NULL)
-				break;
-			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	/* Go though all the bans to test the objects */
+	VTAILQ_FOREACH_REVERSE(bt, &ban_head, banhead_s, list) {
+		if (bt == VTAILQ_LAST(&obans, banhead_s)) {
 			if (DO_DEBUG(DBG_LURKER))
-				VSLb(vsl, SLT_Debug, "test: %p %u %u",
-				    oc, oc->flags & OC_F_LURK, pass);
-			if ((oc->flags & OC_F_LURK) == pass)
-				break;
-			oh = oc->objhead;
-			CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-			if (Lck_Trylock(&oh->mtx)) {
-				Lck_Unlock(&ban_mtx);
-				VSL_Flush(vsl, 0);
-				VTIM_sleep(cache_param->ban_lurker_sleep);
-				continue;
-			}
-			/*
-			 * See if the objcore is still on the objhead since
-			 * we race against HSH_Deref() which comes in the
-			 * opposite locking order.
-			 */
-			VTAILQ_FOREACH(oc2, &oh->objcs, list)
-				if (oc == oc2)
-					break;
-			if (oc2 == NULL) {
-				Lck_Unlock(&oh->mtx);
-				Lck_Unlock(&ban_mtx);
-				VTIM_sleep(cache_param->ban_lurker_sleep);
-				continue;
-			}
-			/*
-			 * If the object is busy, we can't touch
-			 * it. Defer it to a later run.
-			 */
-			if (oc->flags & OC_F_BUSY) {
-				oc->flags |= pass;
-				VTAILQ_REMOVE(&b->objcore, oc, ban_list);
-				VTAILQ_INSERT_TAIL(&b->objcore, oc, ban_list);
-				Lck_Unlock(&oh->mtx);
-				Lck_Unlock(&ban_mtx);
-				continue;
-			}
-			/*
-			 * Grab a reference to the OC and we can let go of
-			 * the BAN mutex
-			 */
-			AN(oc->refcnt);
-			oc->refcnt++;
-			oc->flags &= ~OC_F_LURK;
+				VSLb(vsl, SLT_Debug, "Lurk bt gone %p", bt);
+			Lck_Lock(&ban_mtx);
+			ban_mark_gone(bt);
 			Lck_Unlock(&ban_mtx);
-			/*
-			 * Get the object and check it against all relevant bans
-			 */
-			o = oc_getobj(&wrk->stats, oc);
-			i = ban_check_object(o, vsl, NULL);
-			if (DO_DEBUG(DBG_LURKER))
-				VSLb(vsl, SLT_Debug, "lurker got: %p %d",
-				    oc, i);
-			if (i == -1) {
-				/* Not banned, not moved */
-				oc->flags |= pass;
-				Lck_Lock(&ban_mtx);
-				VTAILQ_REMOVE(&b->objcore, oc, ban_list);
-				VTAILQ_INSERT_TAIL(&b->objcore, oc, ban_list);
-				Lck_Unlock(&ban_mtx);
-			}
-			Lck_Unlock(&oh->mtx);
-			if (DO_DEBUG(DBG_LURKER))
-				VSLb(vsl, SLT_Debug, "lurker done: %p %u %u",
-				    oc, oc->flags & OC_F_LURK, pass);
-			(void)HSH_DerefObj(&wrk->stats, &o);
-			VTIM_sleep(cache_param->ban_lurker_sleep);
-		}
-		Lck_AssertHeld(&ban_mtx);
-		if (!(b->flags & BANS_FLAG_REQ)) {
-			if (!(b->flags & BANS_FLAG_GONE))
-				ban_mark_gone(b);
-			if (DO_DEBUG(DBG_LURKER))
-				VSLb(vsl, SLT_Debug, "lurker BAN %f now gone",
-				    ban_time(b->spec));
+			VTAILQ_REMOVE(&obans, bt, l_list);
+			if (VTAILQ_EMPTY(&obans))
+				break;
 		}
-		Lck_Unlock(&ban_mtx);
-		if (ban_shutdown)
-			break;
-		VTIM_sleep(cache_param->ban_lurker_sleep);
-		if (b == b0)
-			break;
+		if (DO_DEBUG(DBG_LURKER))
+			VSLb(vsl, SLT_Debug, "Lurk bt %p", bt);
+		ban_lurker_test_ban(wrk, vsl, bt, &obans);
 	}
-	pass += (1 << LURK_SHIFT);
-	pass &= BAN_F_LURK;
-	if (pass == 0)
-		pass += (1 << LURK_SHIFT);
 	return (1);
 }
 
@@ -1286,9 +1303,17 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 
 	VCLI_Out(cli, "Present bans:\n");
 	VTAILQ_FOREACH(b, &ban_head, list) {
-		VCLI_Out(cli, "%10.6f %5u%s\t", ban_time(b->spec),
+		VCLI_Out(cli, "%10.6f %5u%s", ban_time(b->spec),
 		    bl == b ? b->refcount - 1 : b->refcount,
-		    b->flags & BANS_FLAG_GONE ? "G" : " ");
+		    b->flags & BANS_FLAG_GONE ? "G" : "-");
+		if (DO_DEBUG(DBG_LURKER)) {
+			VCLI_Out(cli, "%s%s%s %p ",
+			    b->flags & BANS_FLAG_REQ ? "R" : "-",
+			    b->flags & BANS_FLAG_OBJ ? "O" : "-",
+			    b->flags & BANS_FLAG_ERROR ? "E" : "-",
+			    b);
+		}
+		VCLI_Out(cli, "  ");
 		ban_render(cli, b->spec);
 		VCLI_Out(cli, "\n");
 		if (VCLI_Overflow(cli))
@@ -1297,7 +1322,7 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 			Lck_Lock(&ban_mtx);
 			struct objcore *oc;
 			VTAILQ_FOREACH(oc, &b->objcore, ban_list)
-				VCLI_Out(cli, "     %p\n", oc);
+				VCLI_Out(cli, "  oc = %p\n", oc);
 			Lck_Unlock(&ban_mtx);
 		}
 	}
@@ -1317,14 +1342,13 @@ BAN_Init(void)
 
 	Lck_New(&ban_mtx, lck_ban);
 	CLI_AddFuncs(ban_cmds);
-	assert(BAN_F_LURK == OC_F_LURK);
-	AN((1 << LURK_SHIFT) & BAN_F_LURK);
-	AN((2 << LURK_SHIFT) & BAN_F_LURK);
 
 	ban_magic = BAN_New();
 	AN(ban_magic);
 	AZ(BAN_Insert(ban_magic));
+	Lck_Lock(&ban_mtx);
 	ban_mark_gone(ban_magic);
+	Lck_Unlock(&ban_mtx);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc
index 2e31705..3ca2b18 100644
--- a/bin/varnishtest/tests/c00049.vtc
+++ b/bin/varnishtest/tests/c00049.vtc
@@ -1,133 +1,184 @@
-varnishtest "ban lurker test"
-
+varnishtest "New ban-lurker test"
 
 server s1 {
 	rxreq
-	expect req.url == "/alpha"
-	txresp -hdr "Foo: /alpha"
+	expect req.url == /1
+	txresp -hdr "Foo: bar1"
 
 	rxreq
-	expect req.url == "/beta"
-	txresp -hdr "Foo: /beta"
+	expect req.url == /2
+	txresp -hdr "Foo: bar2"
 
 	rxreq
-	expect req.url == "/gamma"
-	txresp -hdr "Foo: /gamma"
+	expect req.url == /3
+	txresp -hdr "Foo: bar3"
 
 	rxreq
-	expect req.url == "/delta"
-	txresp -hdr "Foo: /delta"
+	expect req.url == /4
+	txresp -hdr "Foo: bar4"
 
 	rxreq
-	expect req.url == "/alpha"
-	txresp -hdr "Foo: /alpha2"
+	expect req.url == /5
+	txresp -hdr "Foo: bar5"
 
 	rxreq
-	expect req.url == "/beta"
-	txresp -hdr "Foo: /beta2"
+	expect req.url == /6
+	txresp -hdr "Foo: bar6"
 
 	rxreq
-	expect req.url == "/delta"
-	txresp -hdr "Foo: /delta2"
+	expect req.url == /7
+	txresp -hdr "Foo: bar7"
 
-} -start
+	rxreq
+	expect req.url == /4
+	txresp -hdr "Foo: bar4.1"
 
-varnish v1 -vcl+backend {
 } -start
 
+varnish v1 -vcl+backend {} -start
+
 varnish v1 -cliok "param.set ban_lurker_sleep 0"
 varnish v1 -cliok "param.set debug +lurker"
 
-varnish v1 -cliok "ban.list"
 
 client c1 {
-	txreq -url "/alpha"
+	txreq -url /1
 	rxresp
-	expect resp.http.foo == /alpha
+	expect resp.http.foo == bar1
+
+	txreq -url /2
+	rxresp
+	expect resp.http.foo == bar2
 } -run
 
-delay 0.1
-varnish v1 -cliok "ban req.url == /alpha"
-varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 2
-varnish v1 -expect bans_gone == 1
+varnish v1 -cliok "ban obj.http.foo == bar1"
 
 client c1 {
-	txreq -url "/beta"
+	txreq -url /3
 	rxresp
-	expect resp.http.foo == /beta
+	expect resp.http.foo == bar3
 } -run
 
-delay 0.1
-varnish v1 -cliok "ban obj.http.foo == /beta"
-varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 3
+varnish v1 -cliok "ban obj.http.foo == bar2 && obj.http.foo != foof"
 
 client c1 {
-	txreq -url "/gamma"
+	txreq -url /4
 	rxresp
-	expect resp.http.foo == /gamma
+	expect resp.http.foo == bar4
 } -run
 
-delay 0.1
-varnish v1 -cliok "ban obj.http.foo == /gamma"
-varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 4
+varnish v1 -cliok "ban req.http.kill == yes"
 
 client c1 {
-	txreq -url "/delta"
+	txreq -url /5
 	rxresp
-	expect resp.http.foo == /delta
+	expect resp.http.foo == bar5
 } -run
 
-delay 0.1
-varnish v1 -cliok "ban req.url == /delta"
+varnish v1 -cliok "ban obj.http.foo == bar5"
 
-varnish v1 -expect bans_gone == 1
-varnish v1 -cliok "ban obj.http.foo == /gamma"
-# Dup-check should have added one
-varnish v1 -expect bans_gone == 2
+client c1 {
+	txreq -url /6
+	rxresp
+	expect resp.http.foo == bar6
+} -run
+
+varnish v1 -cliok "ban obj.http.foo == bar6"
+
+client c1 {
+	txreq -url /7
+	rxresp
+	expect resp.http.foo == bar7
+} -run
+
+# Get the VSL out of the way
+delay .1
 
-varnish v1 -cliok "ban req.url == /epsilon"
 varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 7
-varnish v1 -expect bans_gone == 2
+
+varnish v1 -expect bans == 6
+varnish v1 -expect bans_gone == 1
+varnish v1 -expect bans_req == 1
+varnish v1 -expect bans_obj == 4
+varnish v1 -expect bans_added == 6
+varnish v1 -expect bans_deleted == 0
+varnish v1 -expect bans_tested == 0
+varnish v1 -expect bans_tests_tested == 0
+varnish v1 -expect bans_obj_killed == 0
+varnish v1 -expect bans_lurker_tested == 0
+varnish v1 -expect bans_lurker_tests_tested == 0
+varnish v1 -expect bans_lurker_obj_killed == 0
+varnish v1 -expect bans_dups == 0
+
 
 varnish v1 -cliok "param.set ban_lurker_sleep .01"
-delay 1
-varnish v1 -cliok "param.set ban_lurker_sleep .00"
+
+delay 2
+
 varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 7
+
+varnish v1 -expect bans == 5
 varnish v1 -expect bans_gone == 4
+varnish v1 -expect bans_req == 1
+varnish v1 -expect bans_obj == 4
+varnish v1 -expect bans_added == 6
+varnish v1 -expect bans_deleted == 1
+varnish v1 -expect bans_tested == 0
+varnish v1 -expect bans_tests_tested == 0
+varnish v1 -expect bans_obj_killed == 0
+varnish v1 -expect bans_lurker_tested == 10
+varnish v1 -expect bans_lurker_tests_tested == 11
+varnish v1 -expect bans_lurker_obj_killed == 4
+varnish v1 -expect bans_dups == 0
 
 client c1 {
-	txreq -url "/alpha"
+	txreq -url /3
 	rxresp
-	expect resp.http.foo == /alpha2
+	expect resp.http.foo == bar3
 } -run
 
+# Give lurker time to trim tail
 delay 1
-varnish v1 -cliok "ban.list"
-varnish v1 -expect bans == 4
-
-client c1 {
-	txreq -url "/beta"
-	rxresp
-	expect resp.http.foo == /beta2
-} -run
 
 varnish v1 -cliok "ban.list"
 
+varnish v1 -expect bans == 4
+varnish v1 -expect bans_gone == 3
+varnish v1 -expect bans_req == 1
+varnish v1 -expect bans_obj == 3
+varnish v1 -expect bans_added == 6
+varnish v1 -expect bans_deleted == 2
+varnish v1 -expect bans_tested == 1
+varnish v1 -expect bans_tests_tested == 1
+varnish v1 -expect bans_obj_killed == 0
+varnish v1 -expect bans_lurker_tested == 10
+varnish v1 -expect bans_lurker_tests_tested == 11
+varnish v1 -expect bans_lurker_obj_killed == 4
+varnish v1 -expect bans_dups == 0
 
 client c1 {
-	txreq -url "/delta"
+	txreq -url /4 -hdr "kill: yes"
 	rxresp
-	expect resp.http.foo == /delta2
+	expect resp.http.foo == bar4.1
 } -run
 
+# Give lurker time to trim tail
 delay 1
+
 varnish v1 -cliok "ban.list"
+
 varnish v1 -expect bans == 1
-varnish v1 -expect bans_gone == 0
-varnish v1 -expect bans_added == 7
-varnish v1 -expect bans_deleted == 6
+varnish v1 -expect bans_gone == 1
+varnish v1 -expect bans_req == 0
+varnish v1 -expect bans_obj == 1
+varnish v1 -expect bans_added == 6
+varnish v1 -expect bans_deleted == 5
+varnish v1 -expect bans_tested == 2
+varnish v1 -expect bans_tests_tested == 2
+varnish v1 -expect bans_obj_killed == 1
+varnish v1 -expect bans_lurker_tested == 10
+varnish v1 -expect bans_lurker_tests_tested == 11
+varnish v1 -expect bans_lurker_obj_killed == 4
+varnish v1 -expect bans_dups == 0
+
+varnish v1 -expect n_object == 3
diff --git a/bin/varnishtest/tests/r01030.vtc b/bin/varnishtest/tests/r01030.vtc
index 8b7d403..ca7ef88 100644
--- a/bin/varnishtest/tests/r01030.vtc
+++ b/bin/varnishtest/tests/r01030.vtc
@@ -40,10 +40,10 @@ client c1 {
 } -run
 
 #delay 0.1
-varnish v1 -expect bans_tests_tested == 0
+varnish v1 -expect bans_lurker_tests_tested == 0
 
 delay 1.0
-varnish v1 -expect bans_tests_tested == 1
+varnish v1 -expect bans_lurker_tests_tested == 1
 
 varnish v1 -cliok "param.set ban_lurker_sleep 5.01"
 
@@ -58,7 +58,7 @@ client c2 {
 } -run
 
 #delay 0.1
-varnish v1 -expect bans_tests_tested == 1
+varnish v1 -expect bans_lurker_tests_tested == 1
 
 delay 1.1
-varnish v1 -expect bans_tests_tested == 2
+varnish v1 -expect bans_lurker_tests_tested == 2
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 825d73e..121eaae 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -490,6 +490,11 @@ VSC_F(bans_gone,		uint64_t, 0, 'g', diag,
 	"Number of bans which are no longer active, either because they"
 	" got checked by the ban-lurker or superseded by newer identical bans."
 )
+VSC_F(bans_obj,			uint64_t, 0, 'g', diag,
+    "Number of bans using obj.*",
+	"Number of bans which use obj.* variables.  These bans can possibly"
+	" be washed by the ban-lurker."
+)
 VSC_F(bans_req,			uint64_t, 0, 'g', diag,
     "Number of bans using req.*",
 	"Number of bans which use req.* variables.  These bans can not"
@@ -505,20 +510,45 @@ VSC_F(bans_deleted,		uint64_t, 0, 'c', diag,
 )
 
 VSC_F(bans_tested,		uint64_t, 0, 'c', diag,
-    "Bans tested against objects",
+    "Bans tested against objects (lookup)",
+	"Count of how many bans and objects have been tested against"
+	" each other during hash lookup."
+)
+VSC_F(bans_obj_killed,	uint64_t, 0, 'c', diag,
+    "Objects killed by bans (lookup)",
+	"Number of objects killed by bans during object lookup."
+)
+VSC_F(bans_lurker_tested,	uint64_t, 0, 'c', diag,
+    "Bans tested against objects (lurker)",
 	"Count of how many bans and objects have been tested against"
-	" each other."
+	" each other by the ban-lurker."
 )
 VSC_F(bans_tests_tested,	uint64_t, 0, 'c', diag,
-    "Ban tests tested against objects",
+    "Ban tests tested against objects (lookup)",
 	"Count of how many tests and objects have been tested against"
-	" each other.  'ban req.url == foo && req.http.host == bar'"
+	" each other during lookup."
+	"  'ban req.url == foo && req.http.host == bar'"
 	" counts as one in 'bans_tested' and as two in 'bans_tests_tested'"
 )
+VSC_F(bans_lurker_tests_tested,	uint64_t, 0, 'c', diag,
+    "Ban tests tested against objects (lurker)",
+	"Count of how many tests and objects have been tested against"
+	" each other by the ban-lurker."
+	"  'ban req.url == foo && req.http.host == bar'"
+	" counts as one in 'bans_tested' and as two in 'bans_tests_tested'"
+)
+VSC_F(bans_lurker_obj_killed,	uint64_t, 0, 'c', diag,
+    "Objects killed by bans (lurker)",
+	"Number of objects killed by ban-lurker."
+)
 VSC_F(bans_dups,		uint64_t, 0, 'c', diag,
     "Bans superseded by other bans",
 	"Count of bans replaced by later identical bans."
 )
+VSC_F(bans_lurker_contention,	uint64_t, 0, 'c', diag,
+    "Lurker gave way for lookup",
+	"Number of times the ban-lurker had to wait for lookups."
+)
 VSC_F(bans_persisted_bytes,	uint64_t, 0, 'g', diag,
     "Bytes used by the persisted ban lists",
         "Number of bytes used by the persisted ban lists."



From phk at FreeBSD.org  Thu Dec 12 14:53:59 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 12 Dec 2013 15:53:59 +0100
Subject: [master] d7942b7 Fix a (very rare) mtx-leak introduced by previous
 commit.
Message-ID: <E1Vr7eJ-0005XL-3u@project.varnish-software.com>


commit d7942b7efb2925b95a9224b22e9e24bb7e64536d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 12 14:52:56 2013 +0000

    Fix a (very rare) mtx-leak introduced by previous commit.
    
    Spotted by: 	martin
    
    Eliminate some dead code and make sure that the tail-pruner runs,
    also when the lurker is busy.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 1edad6e..d1d0dbd 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -896,10 +896,10 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	struct ban *b;
 	struct objcore *oc;
 	struct ban * volatile b0;
-	unsigned tests, skipped;
+	unsigned tests;
 
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	CHECK_OBJ_ORNULL(req_http, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(req_http, HTTP_MAGIC);
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(oc->ban, BAN_MAGIC);
@@ -925,18 +925,11 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	 * inspect the list past that ban.
 	 */
 	tests = 0;
-	skipped = 0;
 	for (b = b0; b != oc->ban; b = VTAILQ_NEXT(b, list)) {
 		CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
 		if (b->flags & BANS_FLAG_GONE)
 			continue;
-		if (req_http == NULL && (b->flags & BANS_FLAG_REQ)) {
-			/*
-			 * We cannot test this one, but there might
-			 * be other bans that match, so we soldier on
-			 */
-			skipped++;
-		} else if (ban_evaluate(b->spec, o->http, req_http, &tests))
+		if (ban_evaluate(b->spec, o->http, req_http, &tests))
 			break;
 	}
 
@@ -944,16 +937,6 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	VSC_C_main->bans_tested++;
 	VSC_C_main->bans_tests_tested += tests;
 
-	if (b == oc->ban && skipped > 0) {
-		AZ(req_http);
-		Lck_Unlock(&ban_mtx);
-		/*
-		 * Not banned, but some tests were skipped, so we cannot know
-		 * for certain that it cannot be, so we just have to give up.
-		 */
-		return (-1);
-	}
-
 	oc->ban->refcount--;
 	VTAILQ_REMOVE(&oc->ban->objcore, oc, ban_list);
 	if (b == oc->ban) {	/* not banned */
@@ -1038,17 +1021,21 @@ ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt)
 		}
 		oh = oc->objhead;
 		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-		if (!Lck_Trylock(&oh->mtx) && oc->refcnt > 0) {
-			/*
-			 * We got the lock, and the oc is not being
-			 * dismantled under our feet, run with it...
-			 */
-			oc->refcnt += 1;
-			VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
-			VTAILQ_INSERT_TAIL(&bt->objcore, oc, ban_list);
-			Lck_Unlock(&oh->mtx);
-			Lck_Unlock(&ban_mtx);
-			break;
+		if (!Lck_Trylock(&oh->mtx)) {
+			if (oc->refcnt == 0) {
+				Lck_Unlock(&oh->mtx);
+			} else {
+				/*
+				 * We got the lock, and the oc is not being
+				 * dismantled under our feet, run with it...
+				 */
+				oc->refcnt += 1;
+				VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
+				VTAILQ_INSERT_TAIL(&bt->objcore, oc, ban_list);
+				Lck_Unlock(&oh->mtx);
+				Lck_Unlock(&ban_mtx);
+				break;
+			}
 		}
 
 		/* Try again, later */
@@ -1171,30 +1158,23 @@ ban_lurker(struct worker *wrk, void *priv)
 {
 	struct vsl_log vsl;
 	volatile double d;
-	int i = 0, n = 0;
+	int i;
 
 	VSL_Setup(&vsl, NULL, 0);
 
 	(void)priv;
 	while (!ban_shutdown) {
+		i = 0;
 		d = cache_param->ban_lurker_sleep;
-		if (d > 0.0) {
+		if (d > 0.0)
 			i = ban_lurker_work(wrk, &vsl);
-			VSL_Flush(&vsl, 0);
-			WRK_SumStat(wrk);
-			if (i && !ban_shutdown) {
-				VTIM_sleep(d);
-				if (++n > 10) {
-					ban_cleantail();
-					n = 0;
-				}
-				continue;
-			}
-		}
 		ban_cleantail();
 		if (ban_shutdown)
 			break;
-		VTIM_sleep(0.609);	// Random, non-magic
+		if (i)
+			VTIM_sleep(d);
+		else
+			VTIM_sleep(0.609);	// Random, non-magic
 	}
 
 	pthread_exit(0);



From phk at FreeBSD.org  Thu Dec 12 15:18:38 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 12 Dec 2013 16:18:38 +0100
Subject: [master] 295e99a Change "GONE"  bans to "COMPLETED" bans.
Message-ID: <E1Vr82A-0000Xr-GD@project.varnish-software.com>


commit 295e99af97886ee26ae653f1dd67312fcebfd464
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 12 15:18:20 2013 +0000

    Change "GONE"  bans to "COMPLETED" bans.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index d1d0dbd..70ed654 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -41,7 +41,7 @@
  * Bans are compiled into bytestrings as follows:
  *	8 bytes	- double: timestamp		XXX: Byteorder ?
  *	4 bytes - be32: length
- *	1 byte - flags: 0x01: BAN_F_{REQ|OBJ|GONE}
+ *	1 byte - flags: 0x01: BAN_F_{REQ|OBJ|COMPLETED}
  *	N tests
  * A test have this form:
  *	1 byte - arg (see ban_vars.h col 3 "BANS_ARG_XXX")
@@ -88,7 +88,7 @@
 
 #define BANS_FLAG_REQ		(1<<0)
 #define BANS_FLAG_OBJ		(1<<1)
-#define BANS_FLAG_GONE		(1<<2)
+#define BANS_FLAG_COMPLETED	(1<<2)
 #define BANS_FLAG_HTTP		(1<<3)
 #define BANS_FLAG_ERROR		(1<<4)
 
@@ -268,7 +268,7 @@ ban_equal(const uint8_t *bs1, const uint8_t *bs2)
 }
 
 static void
-ban_mark_gone(struct ban *b)
+ban_mark_completed(struct ban *b)
 {
 	unsigned ln;
 
@@ -276,13 +276,13 @@ ban_mark_gone(struct ban *b)
 	Lck_AssertHeld(&ban_mtx);
 
 	AN(b->spec);
-	AZ(b->flags & BANS_FLAG_GONE);
+	AZ(b->flags & BANS_FLAG_COMPLETED);
 	ln = ban_len(b->spec);
-	b->flags |= BANS_FLAG_GONE;
-	b->spec[BANS_FLAGS] |= BANS_FLAG_GONE;
+	b->flags |= BANS_FLAG_COMPLETED;
+	b->spec[BANS_FLAGS] |= BANS_FLAG_COMPLETED;
 	VWMB();
 	vbe32enc(b->spec + BANS_LENGTH, BANS_HEAD_LEN);
-	VSC_C_main->bans_gone++;
+	VSC_C_main->bans_completed++;
 	VSC_C_main->bans_persisted_fragmentation += ln - ban_len(b->spec);
 }
 
@@ -555,16 +555,16 @@ BAN_Insert(struct ban *b)
 	if (be == NULL)
 		return (NULL);
 
-	/* Hunt down duplicates, and mark them as gone */
+	/* Hunt down duplicates, and mark them as completed */
 	bi = b;
 	Lck_Lock(&ban_mtx);
 	while(!ban_shutdown && bi != be) {
 		bi = VTAILQ_NEXT(bi, list);
-		if (bi->flags & BANS_FLAG_GONE)
+		if (bi->flags & BANS_FLAG_COMPLETED)
 			continue;
 		if (!ban_equal(b->spec, bi->spec))
 			continue;
-		ban_mark_gone(bi);
+		ban_mark_completed(bi);
 		VSC_C_main->bans_dups++;
 	}
 	be->refcount--;
@@ -671,7 +671,7 @@ ban_info(enum baninfo event, const uint8_t *ban, unsigned len)
 	if (STV_BanInfo(event, ban, len)) {
 		/* One or more stevedores reported failure. Export the
 		 * list instead. The exported list should take up less
-		 * space due to drops being purged and gones being
+		 * space due to drops being purged and completed being
 		 * truncated. */
 		/* XXX: Keep some measure of how much space can be
 		 * saved, and only export if it's worth it. Assert if
@@ -684,8 +684,8 @@ ban_info(enum baninfo event, const uint8_t *ban, unsigned len)
  * Put a skeleton ban in the list, unless there is an identical,
  * time & condition, ban already in place.
  *
- * If a newer ban has same condition, mark the new ban GONE.
- * mark any older bans, with the same condition, GONE as well.
+ * If a newer ban has same condition, mark the inserted ban COMPLETED,
+ * also mark any older bans, with the same condition COMPLETED.
  */
 
 static void
@@ -727,8 +727,8 @@ ban_reload(const uint8_t *ban, unsigned len)
 	}
 	if (duplicate)
 		VSC_C_main->bans_dups++;
-	if (duplicate || (ban[BANS_FLAGS] & BANS_FLAG_GONE))
-		ban_mark_gone(b2);
+	if (duplicate || (ban[BANS_FLAGS] & BANS_FLAG_COMPLETED))
+		ban_mark_completed(b2);
 	if (b == NULL)
 		VTAILQ_INSERT_TAIL(&ban_head, b2, list);
 	else
@@ -737,10 +737,10 @@ ban_reload(const uint8_t *ban, unsigned len)
 
 	/* Hunt down older duplicates */
 	for (b = VTAILQ_NEXT(b2, list); b != NULL; b = VTAILQ_NEXT(b, list)) {
-		if (b->flags & BANS_FLAG_GONE)
+		if (b->flags & BANS_FLAG_COMPLETED)
 			continue;
 		if (ban_equal(b->spec, ban)) {
-			ban_mark_gone(b);
+			ban_mark_completed(b);
 			VSC_C_main->bans_dups++;
 		}
 	}
@@ -927,7 +927,7 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 	tests = 0;
 	for (b = b0; b != oc->ban; b = VTAILQ_NEXT(b, list)) {
 		CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
-		if (b->flags & BANS_FLAG_GONE)
+		if (b->flags & BANS_FLAG_COMPLETED)
 			continue;
 		if (ban_evaluate(b->spec, o->http, req_http, &tests))
 			break;
@@ -975,8 +975,8 @@ ban_cleantail(void)
 		Lck_Lock(&ban_mtx);
 		b = VTAILQ_LAST(&ban_head, banhead_s);
 		if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
-			if (b->flags & BANS_FLAG_GONE)
-				VSC_C_main->bans_gone--;
+			if (b->flags & BANS_FLAG_COMPLETED)
+				VSC_C_main->bans_completed--;
 			if (b->flags & BANS_FLAG_OBJ)
 				VSC_C_main->bans_obj--;
 			if (b->flags & BANS_FLAG_REQ)
@@ -1117,7 +1117,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 	Lck_Unlock(&ban_mtx);
 	i = 0;
 	while (b != NULL) {
-		if (b->flags & BANS_FLAG_GONE) {
+		if (b->flags & BANS_FLAG_COMPLETED) {
 			;
 		} else if (b->flags & BANS_FLAG_REQ) {
 			;
@@ -1138,9 +1138,10 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 	VTAILQ_FOREACH_REVERSE(bt, &ban_head, banhead_s, list) {
 		if (bt == VTAILQ_LAST(&obans, banhead_s)) {
 			if (DO_DEBUG(DBG_LURKER))
-				VSLb(vsl, SLT_Debug, "Lurk bt gone %p", bt);
+				VSLb(vsl, SLT_Debug,
+				    "Lurk bt completed %p", bt);
 			Lck_Lock(&ban_mtx);
-			ban_mark_gone(bt);
+			ban_mark_completed(bt);
 			Lck_Unlock(&ban_mtx);
 			VTAILQ_REMOVE(&obans, bt, l_list);
 			if (VTAILQ_EMPTY(&obans))
@@ -1283,9 +1284,9 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 
 	VCLI_Out(cli, "Present bans:\n");
 	VTAILQ_FOREACH(b, &ban_head, list) {
-		VCLI_Out(cli, "%10.6f %5u%s", ban_time(b->spec),
+		VCLI_Out(cli, "%10.6f %5u %s", ban_time(b->spec),
 		    bl == b ? b->refcount - 1 : b->refcount,
-		    b->flags & BANS_FLAG_GONE ? "G" : "-");
+		    b->flags & BANS_FLAG_COMPLETED ? "C" : " ");
 		if (DO_DEBUG(DBG_LURKER)) {
 			VCLI_Out(cli, "%s%s%s %p ",
 			    b->flags & BANS_FLAG_REQ ? "R" : "-",
@@ -1327,7 +1328,7 @@ BAN_Init(void)
 	AN(ban_magic);
 	AZ(BAN_Insert(ban_magic));
 	Lck_Lock(&ban_mtx);
-	ban_mark_gone(ban_magic);
+	ban_mark_completed(ban_magic);
 	Lck_Unlock(&ban_mtx);
 }
 
diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc
index 3ca2b18..70620da 100644
--- a/bin/varnishtest/tests/c00049.vtc
+++ b/bin/varnishtest/tests/c00049.vtc
@@ -97,7 +97,7 @@ delay .1
 varnish v1 -cliok "ban.list"
 
 varnish v1 -expect bans == 6
-varnish v1 -expect bans_gone == 1
+varnish v1 -expect bans_completed == 1
 varnish v1 -expect bans_req == 1
 varnish v1 -expect bans_obj == 4
 varnish v1 -expect bans_added == 6
@@ -118,7 +118,7 @@ delay 2
 varnish v1 -cliok "ban.list"
 
 varnish v1 -expect bans == 5
-varnish v1 -expect bans_gone == 4
+varnish v1 -expect bans_completed == 4
 varnish v1 -expect bans_req == 1
 varnish v1 -expect bans_obj == 4
 varnish v1 -expect bans_added == 6
@@ -143,7 +143,7 @@ delay 1
 varnish v1 -cliok "ban.list"
 
 varnish v1 -expect bans == 4
-varnish v1 -expect bans_gone == 3
+varnish v1 -expect bans_completed == 3
 varnish v1 -expect bans_req == 1
 varnish v1 -expect bans_obj == 3
 varnish v1 -expect bans_added == 6
@@ -168,7 +168,7 @@ delay 1
 varnish v1 -cliok "ban.list"
 
 varnish v1 -expect bans == 1
-varnish v1 -expect bans_gone == 1
+varnish v1 -expect bans_completed == 1
 varnish v1 -expect bans_req == 0
 varnish v1 -expect bans_obj == 1
 varnish v1 -expect bans_added == 6
diff --git a/bin/varnishtest/tests/p00009.vtc b/bin/varnishtest/tests/p00009.vtc
index 47f216d..66b1e2f 100644
--- a/bin/varnishtest/tests/p00009.vtc
+++ b/bin/varnishtest/tests/p00009.vtc
@@ -1,4 +1,4 @@
-varnishtest "Check that reloaded bans with gone flag are really gone on restart"
+varnishtest "Check that reloaded bans with completed flag are really completed on restart"
 
 shell "rm -f ${tmpdir}/_.per[12]"
 
@@ -36,8 +36,8 @@ varnish v1 -expect bans == 3
 # Expect 1 of the 2 added to be marked dup
 varnish v1 -expect bans_dups == 1
 
-# Expect ban_magic plus our 1 dup to be marked gone
-varnish v1 -expect bans_gone == 2
+# Expect ban_magic plus our 1 dup to be marked completed
+varnish v1 -expect bans_completed == 2
 
 # Restart
 varnish v1 -stop
@@ -54,4 +54,4 @@ client c1 {
 # Expect our duplicate
 varnish v1 -expect bans_dups == 1
 # One more than before restart due to the new ban_magic
-varnish v1 -expect bans_gone == 3
+varnish v1 -expect bans_completed == 3
diff --git a/bin/varnishtest/tests/r01266.vtc b/bin/varnishtest/tests/r01266.vtc
index 04e3cb9..76352c3 100644
--- a/bin/varnishtest/tests/r01266.vtc
+++ b/bin/varnishtest/tests/r01266.vtc
@@ -1,6 +1,6 @@
-varnishtest "#1266 - Check persisted truncated gone bans"
+varnishtest "#1266 - Check persisted truncated completed bans"
 
-# Test that bans which has been gone'd, truncated and persisted works
+# Test that bans which has been completed, truncated and persisted works
 
 shell "rm -f ${tmpdir}/_.per1"
 
@@ -17,7 +17,7 @@ varnish v1 \
 	}
 varnish v1 -start
 
-# Add a ban that will (with lurker help) become a truncated gone ban last
+# Add a ban that will (with lurker help) become a truncated completed ban last
 # in the list
 varnish v1 -cliok "ban obj.http.x-foo == bar"
 delay 1
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 121eaae..32e3a62 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -485,8 +485,8 @@ VSC_F(bans,			uint64_t, 0, 'g', info,
 	"Number of all bans in system, including bans superseded"
 	" by newer bans and bans already checked by the ban-lurker."
 )
-VSC_F(bans_gone,		uint64_t, 0, 'g', diag,
-    "Number of bans marked 'gone'",
+VSC_F(bans_completed,		uint64_t, 0, 'g', diag,
+    "Number of bans marked 'completed'",
 	"Number of bans which are no longer active, either because they"
 	" got checked by the ban-lurker or superseded by newer identical bans."
 )
@@ -556,7 +556,7 @@ VSC_F(bans_persisted_bytes,	uint64_t, 0, 'g', diag,
 VSC_F(bans_persisted_fragmentation,	uint64_t, 0, 'g', diag,
     "Extra bytes in persisted ban lists due to fragmentation",
         "Number of extra bytes accumulated through dropped and"
-	" gone bans in the persistent ban lists."
+	" completed bans in the persistent ban lists."
 )
 
 /*--------------------------------------------------------------------*/



From phk at FreeBSD.org  Thu Dec 12 15:59:12 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 12 Dec 2013 16:59:12 +0100
Subject: [master] a190e03 Handle <vqueue.h> at top level
Message-ID: <E1Vr8fQ-0006pg-O3@project.varnish-software.com>


commit a190e03d104e89ca3b4106aa10f259f5ca173bd8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 12 15:58:04 2013 +0000

    Handle <vqueue.h> at top level

diff --git a/bin/flint.lnt b/bin/flint.lnt
index 53f3238..1da0545 100644
--- a/bin/flint.lnt
+++ b/bin/flint.lnt
@@ -24,21 +24,6 @@
 -emacro(731, xxxassert)	// arg to eq/non-eq
 -emacro(527, WRONG)     // unreachable code
 
-///////////////////////////////////////////////////////////////////////
-// <vqueue.h>
-
--esym(755, VLIST_*)	// Global macro not ref.
--esym(755, VSLIST_*)
--esym(755, VSTAILQ_*)
--esym(755, VTAILQ_*)
-
--emacro((826), VTAILQ_LAST) // Suspicious pointer-to-pointer conversion (area too small)
--emacro((826), VTAILQ_PREV) // Suspicious pointer-to-pointer conversion (area too small)
--emacro(506, VTAILQ_FOREACH_SAFE) // constant value boolean
--emacro(506, VSTAILQ_FOREACH_SAFE) // constant value boolean
--emacro(740, VTAILQ_LAST) // Unusual pointer cast (incompatible indirect types)
--emacro(740, VTAILQ_PREV) // Unusual pointer cast (incompatible indirect types)
--esym(755, VTAILQ_*)
 
 ///////////////////////////////////////////////////////////////////////
 // miniobj
diff --git a/flint.lnt b/flint.lnt
index 2ccd107..5366617 100644
--- a/flint.lnt
+++ b/flint.lnt
@@ -5,3 +5,23 @@
 -efile(451, "tbl/*.h")				// No include guard
 -esym(785,VSL_tags)				// Sparse array
 
+///////////////////////////////////////////////////////////////////////
+// <vqueue.h>
+
+-esym(755, VLIST_*)	// Global macro not ref.
+-esym(755, VSLIST_*)
+-esym(755, VSTAILQ_*)
+-esym(755, VTAILQ_*)
+
+// 506 = constant value boolean
+-emacro(506, VTAILQ_FOREACH_REVERSE_SAFE)
+-emacro(506, VTAILQ_FOREACH_SAFE)
+-emacro(506, VSTAILQ_FOREACH_SAFE) // constant value boolean
+
+// 826 = Suspicious pointer-to-pointer conversion (area to o small)
+-emacro((826), VTAILQ_LAST)
+-emacro((826), VTAILQ_PREV)
+
+-emacro(740, VTAILQ_LAST) // Unusual pointer cast (incompatible indirect types)
+-emacro(740, VTAILQ_PREV) // Unusual pointer cast (incompatible indirect types)
+
diff --git a/lib/libvarnish/flint.lnt b/lib/libvarnish/flint.lnt
index 5aafc98..f86bb8d 100644
--- a/lib/libvarnish/flint.lnt
+++ b/lib/libvarnish/flint.lnt
@@ -30,12 +30,6 @@
 -e785	// Too few initializers for aggregate 
 -e786	// String concatenation within initializer
 
--emacro(740, VTAILQ_PREV) // Unusual pointer cast (incompatible indirect types)
--emacro(740, VTAILQ_LAST) // Unusual pointer cast (incompatible indirect types)
--emacro((826), VTAILQ_PREV) // Suspicious pointer-to-pointer conversion (area too small)
--emacro((826), VTAILQ_LAST) // Suspicious pointer-to-pointer conversion (area too small)
--emacro(506, VTAILQ_FOREACH_SAFE) // constant value boolean
-
 -esym(534, sprintf)	// Ignoring return value of function
 -esym(534, asprintf)	// Ignoring return value of function
 -esym(534, printf)	// Ignoring return value of function



From phk at FreeBSD.org  Thu Dec 12 15:59:12 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 12 Dec 2013 16:59:12 +0100
Subject: [master] 58df714 The bans being washed by the ban-lurker can be
 overtaken by new duplicated bans while it runs.
Message-ID: <E1Vr8fQ-0006pm-Pv@project.varnish-software.com>


commit 58df714a418ecf4144156971001938a9f8880257
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 12 15:58:26 2013 +0000

    The bans being washed by the ban-lurker can be overtaken by new
    duplicated bans while it runs.
    
    Stumbled on by:	scoof
    Fixes #1390

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 70ed654..25274fd 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -1049,9 +1049,9 @@ ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt)
 
 static void
 ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
-    const struct banhead_s *obans)
+    struct banhead_s *obans)
 {
-	struct ban *bl;
+	struct ban *bl, *bln;
 	struct objcore *oc;
 	struct object *o;
 	unsigned tests;
@@ -1076,7 +1076,12 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 		o = oc_getobj(&wrk->stats, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		i = 0;
-		VTAILQ_FOREACH_REVERSE(bl, obans, banhead_s, l_list) {
+		VTAILQ_FOREACH_REVERSE_SAFE(bl, obans, banhead_s, l_list, bln) {
+			if (bl->flags & BANS_FLAG_COMPLETED) {
+				/* Ban was overtaken by new (dup) ban */
+				VTAILQ_REMOVE(obans, bl, l_list);
+				continue;
+			}
 			tests = 0;
 			i = ban_evaluate(bl->spec, o->http, NULL, &tests);
 			VSC_C_main->bans_lurker_tested++;
@@ -1101,16 +1106,10 @@ static int
 ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 {
 	struct ban *b, *bt;
-	//struct objhead *oh;
-	// struct objcore *oc, *oc2;
-	//struct object *o;
 	struct banhead_s obans;
 	int i;
 
-	/*
-	 * Make a list of the bans we can do something about
-	 */
-
+	/* Make a list of the bans we can do something about */
 	VTAILQ_INIT(&obans);
 	Lck_Lock(&ban_mtx);
 	b = ban_start;
@@ -1141,7 +1140,9 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 				VSLb(vsl, SLT_Debug,
 				    "Lurk bt completed %p", bt);
 			Lck_Lock(&ban_mtx);
-			ban_mark_completed(bt);
+			/* We can be raced by a new ban */
+			if (!(bt->flags & BANS_FLAG_COMPLETED))
+				ban_mark_completed(bt);
 			Lck_Unlock(&ban_mtx);
 			VTAILQ_REMOVE(&obans, bt, l_list);
 			if (VTAILQ_EMPTY(&obans))
@@ -1150,6 +1151,8 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 		if (DO_DEBUG(DBG_LURKER))
 			VSLb(vsl, SLT_Debug, "Lurk bt %p", bt);
 		ban_lurker_test_ban(wrk, vsl, bt, &obans);
+		if (VTAILQ_EMPTY(&obans))
+			break;
 	}
 	return (1);
 }



From phk at FreeBSD.org  Fri Dec 13 11:15:30 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 13 Dec 2013 12:15:30 +0100
Subject: [master] 61ade8c Change the way the pacing of the ban-lurker is
 configured.
Message-ID: <E1VrQiQ-0003wq-0q@project.varnish-software.com>


commit 61ade8c8e583f0781f8dea72ad726ae758077921
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 13 11:12:17 2013 +0000

    Change the way the pacing of the ban-lurker is configured.
    
    A new parameter, $ban_lurker_age, sets a minimum age bans must
    have before the lurker will touch them.  This way the lurker
    stays out of the "rush" that happens right after a ban has been
    added.
    
    Another new parameter, $ban_lurker_batch, determines how many
    objects the lurker will examine, before it takes a $ban_lurker_sleep
    in order to not monopolize CPU.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 25274fd..426faa7 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -135,6 +135,7 @@ static pthread_t ban_thread;
 static struct ban * volatile ban_start;
 static struct objcore oc_marker = { .magic = OBJCORE_MAGIC, };
 static bgthread_t ban_lurker;
+static unsigned ban_batch;
 static int ban_shutdown = 0;
 
 /*--------------------------------------------------------------------
@@ -1057,7 +1058,6 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 	unsigned tests;
 	int i;
 
-	(void)wrk;
 	/*
 	 * First see if there is anything to do, and if so, insert marker
 	 */
@@ -1070,6 +1070,10 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 		return;
 
 	while (1) {
+		if (++ban_batch > cache_param->ban_lurker_batch) {
+			VTIM_sleep(cache_param->ban_lurker_sleep);
+			ban_batch = 0;
+		}
 		oc = ban_lurker_getfirst(vsl, bt);
 		if (oc == NULL)
 			return;
@@ -1107,6 +1111,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 {
 	struct ban *b, *bt;
 	struct banhead_s obans;
+	double d;
 	int i;
 
 	/* Make a list of the bans we can do something about */
@@ -1115,6 +1120,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 	b = ban_start;
 	Lck_Unlock(&ban_mtx);
 	i = 0;
+	d = VTIM_real() - cache_param->ban_lurker_age;
 	while (b != NULL) {
 		if (b->flags & BANS_FLAG_COMPLETED) {
 			;
@@ -1122,6 +1128,8 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 			;
 		} else if (b == VTAILQ_LAST(&ban_head, banhead_s)) {
 			;
+		} else if (ban_time(b->spec) > d) {
+			;
 		} else {
 			VTAILQ_INSERT_TAIL(&obans, b, l_list);
 			i++;
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index 58c7e9b..7e208af 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -177,8 +177,9 @@ struct params {
 	/* Get rid of duplicate bans */
 	unsigned		ban_dups;
 
-	/* How long time does the ban lurker sleep */
+	double			ban_lurker_age;
 	double			ban_lurker_sleep;
+	unsigned		ban_lurker_batch;
 
 	unsigned		syslog_cli_traffic;
 
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index 6c4e1e1..e63ce33 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -464,6 +464,16 @@ struct parspec mgt_parspec[] = {
 		"Log all CLI traffic to syslog(LOG_INFO).",
 		0,
 		"on", "bool" },
+	{ "ban_lurker_age", tweak_timeout,
+		&mgt_param.ban_lurker_age,
+		"0", NULL,
+		"The ban lurker does not process bans until they are this"
+		" old.  Right when a ban is added, the most frequently hit"
+		" objects will get tested against it as part of object"
+		" lookup.  This parameter prevents the ban-lurker from"
+		" kicking in, until the rush is over.",
+		0,
+		"60", "s" },
 	{ "ban_lurker_sleep", tweak_timeout,
 		&mgt_param.ban_lurker_sleep,
 		"0", NULL,
@@ -474,6 +484,14 @@ struct parspec mgt_parspec[] = {
 		"A value of zero disables the ban lurker.",
 		0,
 		"0.01", "s" },
+	{ "ban_lurker_batch", tweak_uint,
+		&mgt_param.ban_lurker_batch,
+		"1", NULL,
+		"How many objects the ban lurker examines before taking a"
+		" ban_lurker_sleep.  Use this to pace the ban lurker so it"
+		" does not eat too much CPU.",
+		0,
+		"1000", "" },
 	{ "http_range_support", tweak_bool, &mgt_param.http_range_support,
 		NULL, NULL,
 		"Enable support for HTTP Range headers.",
diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc
index 70620da..40f7b94 100644
--- a/bin/varnishtest/tests/c00049.vtc
+++ b/bin/varnishtest/tests/c00049.vtc
@@ -37,6 +37,7 @@ server s1 {
 
 varnish v1 -vcl+backend {} -start
 
+varnish v1 -cliok "param.set ban_lurker_age 0"
 varnish v1 -cliok "param.set ban_lurker_sleep 0"
 varnish v1 -cliok "param.set debug +lurker"
 
diff --git a/bin/varnishtest/tests/r01030.vtc b/bin/varnishtest/tests/r01030.vtc
index ca7ef88..fcc9594 100644
--- a/bin/varnishtest/tests/r01030.vtc
+++ b/bin/varnishtest/tests/r01030.vtc
@@ -25,7 +25,7 @@ varnish v1 -vcl+backend {
 	}
 } -start
 
-varnish v1 -cliok "param.set ban_lurker_sleep 0.01"
+varnish v1 -cliok "param.set ban_lurker_age 0"
 varnish v1 -expect bans_tests_tested == 0
 
 delay 0.01



From phk at FreeBSD.org  Fri Dec 13 12:23:28 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 13 Dec 2013 13:23:28 +0100
Subject: [master] 7078960 Introduce VCL_BLOB type which VMOD functions can use
 to pass random bits of memory to each other.
Message-ID: <E1VrRmC-0002A9-81@project.varnish-software.com>


commit 70789601e2246ceead87f62c17cd1eb8bdb9025d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 13 12:21:45 2013 +0000

    Introduce VCL_BLOB type which VMOD functions can use to pass
    random bits of memory to each other.
    
    I have added a "len" field to the vmod_priv structure and used that
    for BLOB's.
    
    We may need some memory-mgt beauty/convenience functions for this.

diff --git a/bin/varnishtest/tests/m00012.vtc b/bin/varnishtest/tests/m00012.vtc
new file mode 100644
index 0000000..ef07f90
--- /dev/null
+++ b/bin/varnishtest/tests/m00012.vtc
@@ -0,0 +1,44 @@
+varnishtest "Test VMOD BLOBS"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_debug};
+
+	sub vcl_deliver {
+		set resp.http.foo = debug.blob2hex(debug.str2blob("gunk"));
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.foo == 67756e6b
+} -run
+
+delay .1
+
+varnish v1 -errvcl {BLOBs can only be used as arguments to VMOD functions.} {
+
+	backend b1 {.host = "127.0.0.1";}
+
+	import ${vmod_debug};
+
+	sub vcl_deliver {
+		set resp.http.foo = debug.str2blob("gunk");
+	}
+}
+
+varnish v1 -errvcl {Wrong argument type.  Expected BLOB.  Got STRING.} {
+
+	backend b1 {.host = "127.0.0.1";}
+
+	import ${vmod_debug};
+
+	sub vcl_deliver {
+		set resp.http.foo = debug.blob2hex("gunk");
+	}
+}
diff --git a/include/vrt.h b/include/vrt.h
index 6dfefee..ecf0831 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -61,6 +61,7 @@ typedef double				VCL_REAL;
 typedef const char *			VCL_STRING;
 typedef double				VCL_TIME;
 typedef void				VCL_VOID;
+typedef const struct vmod_priv *	VCL_BLOB;
 
 /***********************************************************************
  * This is the composite argument we pass to compiled VCL and VRT
@@ -238,6 +239,7 @@ struct vmod_priv;
 typedef void vmod_priv_free_f(void *);
 struct vmod_priv {
 	void			*priv;
+	int			len;
 	vmod_priv_free_f	*free;
 };
 
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index c868656..f9c5a83 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -392,7 +392,7 @@ vcc_arg_type(const char **p)
  */
 
 static void
-vcc_expr_tostring(struct expr **e, enum var_type fmt)
+vcc_expr_tostring(struct vcc *tl, struct expr **e, enum var_type fmt)
 {
 	const char *p;
 	uint8_t	constant = EXPR_VAR;
@@ -423,6 +423,13 @@ vcc_expr_tostring(struct expr **e, enum var_type fmt)
 	case STRING:
 	case STRING_LIST:
 			break;
+	case BLOB:
+			VSB_printf(tl->sb,
+			    "Wrong use of BLOB value.\n"
+			    "BLOBs can only be used as arguments to VMOD"
+			    " functions.\n");
+			vcc_ErrWhere2(tl, (*e)->t1, tl->t);
+			return;
 	default:
 			INCOMPL();
 			break;
@@ -451,8 +458,10 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym)
 	vcc_expr0(tl, &e2, STRING);
 	if (e2 == NULL)
 		return;
-	if (e2->fmt != STRING)
-		vcc_expr_tostring(&e2, STRING);
+	if (e2->fmt != STRING) {
+		vcc_expr_tostring(tl, &e2, STRING);
+		ERRCHK(tl);
+	}
 
 	SkipToken(tl, ',');
 	ExpectErr(tl, CSTR);
@@ -466,8 +475,10 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym)
 	vcc_expr0(tl, &e2, STRING);
 	if (e2 == NULL)
 		return;
-	if (e2->fmt != STRING)
-		vcc_expr_tostring(&e2, STRING);
+	if (e2->fmt != STRING) {
+		vcc_expr_tostring(tl, &e2, STRING);
+		ERRCHK(tl);
+	}
 	*e = vcc_expr_edit(STRING, "\v1,\n\v2)\v-", *e, e2);
 	SkipToken(tl, ')');
 }
@@ -817,8 +828,10 @@ vcc_expr_string_add(struct vcc *tl, struct expr **e)
 		vcc_NextToken(tl);
 		vcc_expr_mul(tl, &e2, STRING);
 		ERRCHK(tl);
-		if (e2->fmt != STRING && e2->fmt != STRING_LIST)
-			vcc_expr_tostring(&e2, f2);
+		if (e2->fmt != STRING && e2->fmt != STRING_LIST) {
+			vcc_expr_tostring(tl, &e2, f2);
+			ERRCHK(tl);
+		}
 		ERRCHK(tl);
 		assert(e2->fmt == STRING || e2->fmt == STRING_LIST);
 
@@ -857,7 +870,8 @@ vcc_expr_add(struct vcc *tl, struct expr **e, enum var_type fmt)
 
 	/* Unless we specifically ask for a HEADER, fold them to string here */
 	if (fmt != HEADER && f2 == HEADER) {
-		vcc_expr_tostring(e, STRING);
+		vcc_expr_tostring(tl, e, STRING);
+		ERRCHK(tl);
 		f2 = (*e)->fmt;
 		assert(f2 == STRING);
 	}
@@ -1184,8 +1198,11 @@ vcc_Expr(struct vcc *tl, enum var_type fmt)
 	t1 = tl->t;
 	vcc_expr0(tl, &e, fmt);
 	ERRCHK(tl);
-	if (fmt == STRING || fmt == STRING_LIST)
-		vcc_expr_tostring(&e, fmt);
+	e->t1 = t1;
+	if (fmt == STRING || fmt == STRING_LIST) {
+		vcc_expr_tostring(tl, &e, fmt);
+		ERRCHK(tl);
+	}
 	if (!tl->err && fmt != e->fmt)  {
 		VSB_printf(tl->sb, "Expression has type %s, expected %s\n",
 		    vcc_Type(e->fmt), vcc_Type(fmt));
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index af52ec5..6249b36 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -60,6 +60,7 @@ ctypes = {
 	'STRING_LIST':	"const char *, ...",
 	'TIME':		"VCL_TIME",
 	'VOID':		"VCL_VOID",
+	'BLOB':		"VCL_BLOB",
 }
 
 #######################################################################
@@ -381,7 +382,7 @@ class func(object):
 		fo.write(s + ")\n")
 		for i in self.doc_str:
 			fo.write(i + "\n")
-		
+
 
 #######################################################################
 
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 69bfb38..f24c302 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -53,6 +53,14 @@ $Function VOID test_priv_vcl(PRIV_VCL)
 
 Test function for VCL private pointers
 
+$Function BLOB str2blob(STRING)
+
+Turn a string into a blob
+
+$Function STRING blob2hex(BLOB)
+
+Hexdump a blob
+
 $Object obj(STRING) 
 
 Test object
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index e864880..4031e6e 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -29,6 +29,7 @@
 #include "config.h"
 
 #include <stdlib.h>
+#include <stdio.h>
 
 #include "cache/cache.h"
 
@@ -94,3 +95,38 @@ vmod_test_priv_vcl(const struct vrt_ctx *ctx, struct vmod_priv *priv)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
         assert(!strcmp(priv->priv, "FOO"));
 }
+
+VCL_BLOB
+vmod_str2blob(const struct vrt_ctx *ctx, VCL_STRING s)
+{
+	struct vmod_priv *p;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	p = (void*)WS_Alloc(ctx->ws, sizeof *p);
+	AN(p);
+	memset(p, 0, sizeof *p);
+	p->len = strlen(s);
+	p->priv = WS_Copy(ctx->ws, s, -1);
+	return (p);
+}
+
+VCL_STRING
+vmod_blob2hex(const struct vrt_ctx *ctx, VCL_BLOB b)
+{
+	char *s, *p;
+	uint8_t *q;
+	int i;
+
+	s = WS_Alloc(ctx->ws, b->len * 2 + 2);
+	AN(s);
+	p = s;
+	q = b->priv;
+	for (i = 0; i < b->len; i++) {
+		sprintf(p, "%02x", *q);
+		p += 2;
+		q += 1;
+	}
+	vmod_priv_fini(b);
+	return (s);
+}
+



From apj at mutt.dk  Fri Dec 13 12:56:30 2013
From: apj at mutt.dk (Andreas Plesner Jacobsen)
Date: Fri, 13 Dec 2013 13:56:30 +0100
Subject: [master] 41c3176 Don't clobber output on HUP
Message-ID: <E1VrSIA-0005pi-MH@project.varnish-software.com>


commit 41c317674a217898f07df02c7441ccb5abf6ed23
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Fri Dec 13 13:56:15 2013 +0100

    Don't clobber output on HUP

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index c1c1e36..c2656de 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -182,7 +182,7 @@ rotateout(void)
 	AN(CTX.w_arg);
 	AN(CTX.fo);
 	fclose(CTX.fo);
-	openout(0);
+	openout(1);
 	AN(CTX.fo);
 	return (0);
 }



From apj at mutt.dk  Fri Dec 13 13:09:00 2013
From: apj at mutt.dk (Andreas Plesner)
Date: Fri, 13 Dec 2013 14:09:00 +0100
Subject: [master] 9d2730d Fix spelling
Message-ID: <E1VrSUG-0007jb-K5@project.varnish-software.com>


commit 9d2730de5b2ad968c9f107e383817dc39d79220d
Author: Andreas Plesner <apj at mutt.dk>
Date:   Fri Dec 13 14:08:54 2013 +0100

    Fix spelling

diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index 4cf04fe..708f46f 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -104,7 +104,7 @@ VUT_g_Arg(const char *arg)
 
 	VUT.g_arg = VSLQ_Name2Grouping(arg, -1);
 	if (VUT.g_arg == -2)
-		VUT_Error(1, "Ambigous grouping type: %s", arg);
+		VUT_Error(1, "Ambiguous grouping type: %s", arg);
 	else if (VUT.g_arg < 0)
 		VUT_Error(1, "Unknown grouping type: %s", arg);
 	return (1);



From apj at mutt.dk  Fri Dec 13 13:09:00 2013
From: apj at mutt.dk (Andreas Plesner)
Date: Fri, 13 Dec 2013 14:09:00 +0100
Subject: [master] 302a21b Don't clobber output on HUP
Message-ID: <E1VrSUG-0007jV-IQ@project.varnish-software.com>


commit 302a21bb3eb8e122a88790ee1e9948bb777e389c
Author: Andreas Plesner <apj at mutt.dk>
Date:   Fri Dec 13 14:08:15 2013 +0100

    Don't clobber output on HUP

diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index d0589d6..3f6bf9f 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -94,7 +94,7 @@ rotateout(void)
 	AN(LOG.w_arg);
 	AN(LOG.fo);
 	fclose(LOG.fo);
-	openout(0);
+	openout(1);
 	AN(LOG.fo);
 	return (0);
 }



From phk at FreeBSD.org  Fri Dec 13 13:33:14 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 13 Dec 2013 14:33:14 +0100
Subject: [master] 21e0139 More paranoia in the fetch processor.
Message-ID: <E1VrSri-0002V7-Pl@project.varnish-software.com>


commit 21e0139ea73aeae2613bed677ea1172650494f7d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 13 13:32:55 2013 +0000

    More paranoia in the fetch processor.

diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index d7c3610..f080bfd 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -100,12 +100,12 @@ VFP_GetStorage(struct busyobj *bo, ssize_t sz)
 	st = STV_alloc(bo, l);
 	if (st == NULL) {
 		(void)VFP_Error(bo, "Could not get storage");
-		return (NULL);
+	} else {
+		AZ(st->len);
+		Lck_Lock(&bo->mtx);
+		VTAILQ_INSERT_TAIL(&obj->store, st, list);
+		Lck_Unlock(&bo->mtx);
 	}
-	AZ(st->len);
-	Lck_Lock(&bo->mtx);
-	VTAILQ_INSERT_TAIL(&obj->store, st, list);
-	Lck_Unlock(&bo->mtx);
 	return (st);
 }
 
@@ -201,35 +201,32 @@ VFP_Fetch_Body(struct busyobj *bo, ssize_t est)
 	}
 
 	do {
+		assert(bo->state != BOS_FAILED);
+		if (st == NULL) {
+			st = VFP_GetStorage(bo, est);
+			est = 0;
+		}
 		if (st == NULL) {
-			l = fetchfrag;
-			if (l == 0) {
-				l = est;
-				est = 0;
-			}
-			if (l == 0)
-				l = cache_param->fetch_chunksize;
-			st = STV_alloc(bo, l);
-			if (st == NULL) {
-				bo->should_close = 1;
-				/* XXX Close VFP stack */
-				(void)VFP_Error(bo, "Out of storage");
-				break;
-			}
-			AZ(st->len);
-			Lck_Lock(&bo->mtx);
-			VTAILQ_INSERT_TAIL(&bo->fetch_obj->store, st, list);
-			Lck_Unlock(&bo->mtx);
+			bo->should_close = 1;
+			(void)VFP_Error(bo, "Out of storage");
+			break;
 		}
+
+		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+		assert(st == VTAILQ_LAST(&bo->fetch_obj->store, storagehead));
 		l = st->space - st->len;
+		assert(bo->state != BOS_FAILED);
 		vfps = VFP_Suck(bo, st->ptr + st->len, &l);
-		if (l > 0)
+		if (l > 0 && vfps != VFP_ERROR) {
+			assert(!VTAILQ_EMPTY(&bo->fetch_obj->store));
 			VBO_extend(bo, l);
+		}
 		if (st->len == st->space)
 			st = NULL;
 	} while (vfps == VFP_OK);
 
 	if (vfps == VFP_ERROR) {
+		assert(bo->state == BOS_FAILED);
 		(void)VFP_Error(bo, "Fetch Pipeline failed to process");
 		bo->should_close = 1;
 	}



From phk at FreeBSD.org  Fri Dec 13 15:24:40 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 13 Dec 2013 16:24:40 +0100
Subject: [master] 59e3bef Add a "inifin" facility to make Init/Fini activities
 happen in the right order: Init is done first to last "inifin",
 Fini is done in opposite order, last to first.
Message-ID: <E1VrUbY-0005vJ-CE@project.varnish-software.com>


commit 59e3bef7519c9e60c222a4ce8e7bbe338983eee5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 13 15:23:13 2013 +0000

    Add a "inifin" facility to make Init/Fini activities happen in the
    right order:  Init is done first to last "inifin", Fini is done
    in opposite order, last to first.
    
    This replaces the several ad-hoc facilities we had.

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 79786ef..99f97eb 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -165,6 +165,7 @@ static void
 parse_new(struct vcc *tl)
 {
 	struct symbol *sy1, *sy2, *sy3;
+	struct inifin *ifp;
 	const char *p, *s_obj, *s_init, *s_struct, *s_fini;
 	char buf1[128];
 	char buf2[128];
@@ -213,7 +214,8 @@ parse_new(struct vcc *tl)
 
 	bprintf(buf1, ", &%s, \"%s\"", sy1->name, sy1->name);
 	vcc_Eval_Func(tl, s_init, buf1, "ASDF", s_init + strlen(s_init) + 1);
-	Fd(tl, 0, "\t%s(&%s);\n", s_fini, sy1->name);
+	ifp = New_IniFin(tl);
+	VSB_printf(ifp->fin, "\t%s(&%s);", s_fini, sy1->name);
 	ExpectErr(tl, ';');
 
 	bprintf(buf1, ", %s", sy1->name);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 6982fb2..ab0d8b2 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -275,6 +275,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be)
 	struct token *t_port = NULL;
 	struct token *t_hosthdr = NULL;
 	struct fld_spec *fs;
+	struct inifin *ifp;
 	struct vsb *vsb;
 	unsigned u;
 	double t;
@@ -420,9 +421,11 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be)
 	Fh(tl, 0, "%s", VSB_data(vsb));
 	VSB_delete(vsb);
 
-	Fi(tl, 0, "\tVRT_init_dir(cli, VCL_conf.director,\n"
-	    "\t    VGC_backend_%s, &vgc_dir_priv_%s);\n", vgcname, vgcname);
-	Ff(tl, 0, "\tVRT_fini_dir(cli, VGCDIR(%s));\n", vgcname);
+	ifp = New_IniFin(tl);
+	VSB_printf(ifp->ini,
+	    "\tVRT_init_dir(cli, VCL_conf.director,\n"
+	    "\t    VGC_backend_%s, &vgc_dir_priv_%s);", vgcname, vgcname);
+	VSB_printf(ifp->fin, "\tVRT_fini_dir(cli, VGCDIR(%s));", vgcname);
 	tl->ndirector++;
 }
 
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index a1d3b9b..7fa31aa 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -123,6 +123,23 @@ TlDupTok(struct vcc *tl, const struct token *tok)
 
 /*--------------------------------------------------------------------*/
 
+struct inifin *
+New_IniFin(struct vcc *tl)
+{
+	struct inifin *p;
+
+	p = TlAlloc(tl, sizeof *p);
+	AN(p);
+	p->magic = INIFIN_MAGIC;
+	p->ini = VSB_new_auto();
+	p->fin = VSB_new_auto();
+	p->n = ++tl->ninifin;
+	VTAILQ_INSERT_TAIL(&tl->inifin, p, list);
+	return (p);
+}
+
+/*--------------------------------------------------------------------*/
+
 int
 IsMethod(const struct token *t)
 {
@@ -181,43 +198,6 @@ Fc(const struct vcc *tl, int indent, const char *fmt, ...)
 	va_end(ap);
 }
 
-void
-Fi(const struct vcc *tl, int indent, const char *fmt, ...)
-{
-	va_list ap;
-
-	if (indent)
-		VSB_printf(tl->fi, "%*.*s", tl->iindent, tl->iindent, "");
-	va_start(ap, fmt);
-	VSB_vprintf(tl->fi, fmt, ap);
-	va_end(ap);
-}
-
-void
-Fd(const struct vcc *tl, int indent, const char *fmt, ...)
-{
-	va_list ap;
-
-	if (indent)
-		VSB_printf(tl->fd, "%*.*s", tl->findent, tl->findent, "");
-	va_start(ap, fmt);
-	VSB_vprintf(tl->fd, fmt, ap);
-	va_end(ap);
-}
-
-
-void
-Ff(const struct vcc *tl, int indent, const char *fmt, ...)
-{
-	va_list ap;
-
-	if (indent)
-		VSB_printf(tl->ff, "%*.*s", tl->findent, tl->findent, "");
-	va_start(ap, fmt);
-	VSB_vprintf(tl->ff, fmt, ap);
-	va_end(ap);
-}
-
 /*--------------------------------------------------------------------*/
 
 void
@@ -318,10 +298,16 @@ LocTable(const struct vcc *tl)
 static void
 EmitInitFunc(const struct vcc *tl)
 {
+	struct inifin *p;
 
 	Fc(tl, 0, "\nstatic int\nVGC_Init(struct cli *cli)\n{\n\n");
-	AZ(VSB_finish(tl->fi));
-	VSB_cat(tl->fc, VSB_data(tl->fi));
+	VTAILQ_FOREACH(p, &tl->inifin, list) {
+		AZ(VSB_finish(p->ini));
+		if (VSB_len(p->ini))
+			Fc(tl, 0, "\t/* %u */\n%s\n", p->n, VSB_data(p->ini));
+		VSB_delete(p->ini);
+	}
+
 	Fc(tl, 0, "\treturn(0);\n");
 	Fc(tl, 0, "}\n");
 }
@@ -329,22 +315,17 @@ EmitInitFunc(const struct vcc *tl)
 static void
 EmitFiniFunc(const struct vcc *tl)
 {
-	unsigned u;
+	struct inifin *p;
 
 	Fc(tl, 0, "\nstatic void\nVGC_Fini(struct cli *cli)\n{\n\n");
 
-	AZ(VSB_finish(tl->fd));
-	VSB_cat(tl->fc, VSB_data(tl->fd));
-
-	/*
-	 * We do this here, so we are sure they happen before any
-	 * per-vcl vmod_privs get cleaned.
-	 */
-	for (u = 0; u < tl->nvmodpriv; u++)
-		Fc(tl, 0, "\tvmod_priv_fini(&vmod_priv_%u);\n", u);
+	VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) {
+		AZ(VSB_finish(p->fin));
+		if (VSB_len(p->fin))
+			Fc(tl, 0, "\t/* %u */\n%s\n", p->n, VSB_data(p->fin));
+		VSB_delete(p->fin);
+	}
 
-	AZ(VSB_finish(tl->ff));
-	VSB_cat(tl->fc, VSB_data(tl->ff));
 	Fc(tl, 0, "}\n");
 }
 
@@ -519,6 +500,7 @@ vcc_NewVcc(const struct vcc *tl0)
 		tl->err_unref = 1;
 	}
 	VTAILQ_INIT(&tl->symbols);
+	VTAILQ_INIT(&tl->inifin);
 	VTAILQ_INIT(&tl->membits);
 	VTAILQ_INIT(&tl->tokens);
 	VTAILQ_INIT(&tl->sources);
@@ -534,18 +516,6 @@ vcc_NewVcc(const struct vcc *tl0)
 	tl->fh = VSB_new_auto();
 	assert(tl->fh != NULL);
 
-	/* Init C code */
-	tl->fi = VSB_new_auto();
-	assert(tl->fi != NULL);
-
-	/* Destroy Objects */
-	tl->fd = VSB_new_auto();
-	assert(tl->fd != NULL);
-
-	/* Finish C code */
-	tl->ff = VSB_new_auto();
-	assert(tl->ff != NULL);
-
 	/* body code of methods */
 	for (i = 0; i < VCL_MET_MAX; i++) {
 		tl->fm[i] = VSB_new_auto();
@@ -584,8 +554,6 @@ vcc_DestroyTokenList(struct vcc *tl, char *ret)
 
 	VSB_delete(tl->fh);
 	VSB_delete(tl->fc);
-	VSB_delete(tl->fi);
-	VSB_delete(tl->ff);
 	for (i = 0; i < VCL_MET_MAX; i++)
 		VSB_delete(tl->fm[i]);
 
@@ -603,6 +571,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	struct vcc *tl;
 	struct symbol *sym;
 	const struct var *v;
+	struct inifin *ifp;
 	char *of;
 	int i;
 
@@ -678,7 +647,9 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	}
 
 	/* Configure the default director */
-	Fi(tl, 0, "\tVCL_conf.director[0] = VCL_conf.director[%d];\n",
+	ifp = New_IniFin(tl);
+	VSB_printf(ifp->ini,
+	    "\tVCL_conf.director[0] = VCL_conf.director[%d];",
 	    tl->defaultdir);
 	vcc_AddRef(tl, tl->t_defaultdir, SYM_BACKEND);
 
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index cd72db9..07dbbf3 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -144,6 +144,17 @@ struct symbol {
 
 VTAILQ_HEAD(tokenhead, token);
 
+struct inifin {
+	unsigned		magic;
+#define INIFIN_MAGIC		0x583c274c
+	unsigned		n;
+	struct vsb		*ini;
+	struct vsb		*fin;
+	VTAILQ_ENTRY(inifin)	list;
+};
+
+VTAILQ_HEAD(inifinhead, inifin);
+
 struct vcc {
 	unsigned		magic;
 #define VCC_MAGIC		0x24ad719d
@@ -156,6 +167,9 @@ struct vcc {
 	const struct var	*vars;
 	VTAILQ_HEAD(, symbol)	symbols;
 
+	struct inifinhead	inifin;
+	unsigned		ninifin;
+
 	/* Instance section */
 	struct tokenhead	tokens;
 	VTAILQ_HEAD(, source)	sources;
@@ -165,15 +179,10 @@ struct vcc {
 	struct token		*t;
 	int			indent;
 	int			hindent;
-	int			iindent;
-	int			findent;
 	unsigned		cnt;
 
 	struct vsb		*fc;		/* C-code */
 	struct vsb		*fh;		/* H-code (before C-code) */
-	struct vsb		*fi;		/* Init func code */
-	struct vsb		*fd;		/* Object destructors */
-	struct vsb		*ff;		/* Finish func code */
 	struct vsb		*fb;		/* Body of current sub
 						 * NULL otherwise
 						 */
@@ -192,7 +201,6 @@ struct vcc {
 	struct token		*t_defaultdir;
 
 	unsigned		unique;
-	unsigned		nvmodpriv;
 
 	unsigned		err_unref;
 	unsigned		allow_inline_c;
@@ -237,6 +245,8 @@ void vcc_FieldsOk(struct vcc *tl, const struct fld_spec *fs);
 
 /* vcc_compile.c */
 extern struct method method_tab[];
+struct inifin *New_IniFin(struct vcc *tl);
+
 /*
  * H -> Header, before the C code
  * C -> C-code
@@ -250,12 +260,6 @@ void Fc(const struct vcc *tl, int indent, const char *fmt, ...)
     __printflike(3, 4);
 void Fb(const struct vcc *tl, int indent, const char *fmt, ...)
     __printflike(3, 4);
-void Fi(const struct vcc *tl, int indent, const char *fmt, ...)
-    __printflike(3, 4);
-void Ff(const struct vcc *tl, int indent, const char *fmt, ...)
-    __printflike(3, 4);
-void Fd(const struct vcc *tl, int indent, const char *fmt, ...)
-    __printflike(3, 4);
 void EncToken(struct vsb *sb, const struct token *t);
 int IsMethod(const struct token *t);
 void *TlAlloc(struct vcc *tl, unsigned len);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index f9c5a83..fab381a 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -536,6 +536,7 @@ vcc_func(struct vcc *tl, struct expr **e, const char *cfunc,
 {
 	const char *p, *r;
 	struct expr *e1, *e2;
+	struct inifin *ifp;
 	enum var_type fmt;
 	char buf[32];
 
@@ -557,8 +558,10 @@ vcc_func(struct vcc *tl, struct expr **e, const char *cfunc,
 			    (int) (r - name), name);
 			p += strlen(p) + 1;
 		} else if (fmt == VOID && !strcmp(p, "PRIV_CALL")) {
-			bprintf(buf, "vmod_priv_%u", tl->nvmodpriv++);
+			bprintf(buf, "vmod_priv_%u", tl->unique++);
+			ifp = New_IniFin(tl);
 			Fh(tl, 0, "static struct vmod_priv %s;\n", buf);
+			VSB_printf(ifp->fin, "\tvmod_priv_fini(&%s);", buf);
 			e2 = vcc_mk_expr(VOID, "&%s", buf);
 			p += strlen(p) + 1;
 		} else if (fmt == ENUM) {
diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c
index 88edc2d..4f2b99b 100644
--- a/lib/libvcc/vcc_utils.c
+++ b/lib/libvcc/vcc_utils.c
@@ -51,6 +51,7 @@ vcc_regexp(struct vcc *tl)
 	vre_t *t;
 	const char *error;
 	int erroroffset;
+	struct inifin *ifp;
 
 	Expect(tl, CSTR);
 	if (tl->err)
@@ -69,10 +70,11 @@ vcc_regexp(struct vcc *tl)
 	strcpy(p, buf);
 
 	Fh(tl, 0, "static void *%s;\n", buf);
-	Fi(tl, 0, "\tVRT_re_init(&%s, ",buf);
-	EncToken(tl->fi, tl->t);
-	Fi(tl, 0, ");\n");
-	Ff(tl, 0, "\tVRT_re_fini(%s);\n", buf);
+	ifp = New_IniFin(tl);
+	VSB_printf(ifp->ini, "\tVRT_re_init(&%s, ",buf);
+	EncToken(ifp->ini, tl->t);
+	VSB_printf(ifp->ini, ");");
+	VSB_printf(ifp->fin, "\tVRT_re_fini(%s);", buf);
 	return (p);
 }
 
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index a2dc029..98ffeb2 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -43,6 +43,7 @@ vcc_ParseImport(struct vcc *tl)
 	char fn[1024];
 	char buf[256];
 	struct token *mod, *t1;
+	struct inifin *ifp;
 	const char *modname;
 	const char *proto;
 	const char *abi;
@@ -105,15 +106,23 @@ vcc_ParseImport(struct vcc *tl)
 
 	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
 
-	Fi(tl, 0, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
-	Fi(tl, 0, "\t    &Vmod_%.*s_Func,\n", PF(mod));
-	Fi(tl, 0, "\t    sizeof(Vmod_%.*s_Func),\n", PF(mod));
-	Fi(tl, 0, "\t    \"%.*s\",\n", PF(mod));
-	Fi(tl, 0, "\t    ");
-	EncString(tl->fi, fn, NULL, 0);
-	Fi(tl, 0, ",\n\t    ");
-	Fi(tl, 0, "cli))\n");
-	Fi(tl, 0, "\t\treturn(1);\n");
+	ifp = New_IniFin(tl);
+
+	VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    &Vmod_%.*s_Func,\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    sizeof(Vmod_%.*s_Func),\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    \"%.*s\",\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    ");
+	EncString(ifp->ini, fn, NULL, 0);
+	VSB_printf(ifp->ini, ",\n\t    ");
+	VSB_printf(ifp->ini, "cli))\n");
+	VSB_printf(ifp->ini, "\t\treturn(1);");
+
+	/* XXX: zero the function pointer structure ?*/
+	VSB_printf(ifp->fin, "\tvmod_priv_fini(&vmod_priv_%.*s);", PF(mod));
+	VSB_printf(ifp->fin, "\n\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
+
+	ifp = NULL;
 
 	SkipToken(tl, ';');
 
@@ -178,7 +187,10 @@ vcc_ParseImport(struct vcc *tl)
 			sym->args = p;
 		} else if (!strcmp(p, "INIT")) {
 			p += strlen(p) + 1;
-			Fi(tl, 0, "\t%s(&vmod_priv_%.*s, &VCL_conf);\n",
+			if (ifp == NULL)
+				ifp = New_IniFin(tl);
+			VSB_printf(ifp->ini,
+			    "\t%s(&vmod_priv_%.*s, &VCL_conf);",
 			    p, PF(mod));
 		} else {
 			sym = VCC_AddSymbolStr(tl, p, SYM_FUNC);
@@ -196,8 +208,4 @@ vcc_ParseImport(struct vcc *tl)
 		}
 	}
 	Fh(tl, 0, "\n%s\n", proto);
-
-	/* XXX: zero the function pointer structure ?*/
-	Ff(tl, 0, "\tvmod_priv_fini(&vmod_priv_%.*s);\n", PF(mod));
-	Ff(tl, 0, "\tVRT_Vmod_Fini(&VGC_vmod_%.*s);\n", PF(mod));
 }



From phk at FreeBSD.org  Mon Dec 16 13:31:14 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 16 Dec 2013 14:31:14 +0100
Subject: [master] 781fb81 Allow rxhdrs in client
Message-ID: <E1VsYGQ-0007rK-BG@project.varnish-software.com>


commit 781fb81d8361b1789a6f6a0c0e642e4124e715a9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 16 12:50:58 2013 +0000

    Allow rxhdrs in client

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 63b02e6..6a0db8f 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -809,7 +809,6 @@ cmd_http_rxhdrs(CMD_ARGS)
 	(void)cmd;
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
-	ONLY_SERVER(hp, av);
 	assert(!strcmp(av[0], "rxhdrs"));
 	av++;
 



From phk at FreeBSD.org  Mon Dec 16 13:31:14 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 16 Dec 2013 14:31:14 +0100
Subject: [master] b253bf9 When a streaming delivery of a pass object is
 terminated prematurely, we cannot just throw the storage away,
 we have to wait for the fetch-thread to go away,
 possibly in response to a new "abandon" signal.
Message-ID: <E1VsYGQ-0007rQ-D8@project.varnish-software.com>


commit b253bf9c52929e13f13c65670dd08871feb1f977
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 16 13:28:47 2013 +0000

    When a streaming delivery of a pass object is terminated prematurely,
    we cannot just throw the storage away, we have to wait for the
    fetch-thread to go away, possibly in response to a new "abandon"
    signal.
    
    Spotted first by:	scoof
    Fixes	#1391

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 38ca5e3..5998e38 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -573,6 +573,7 @@ struct busyobj {
 	/* do_pass is our intent, uncacheable is the result */
 	unsigned		do_pass;
 	unsigned		uncacheable;
+	unsigned		abandon;
 
 	/* Timeouts */
 	double			connect_timeout;
diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index f080bfd..006768d 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -201,6 +201,19 @@ VFP_Fetch_Body(struct busyobj *bo, ssize_t est)
 	}
 
 	do {
+		if (bo->abandon) {
+			/*
+			 * A pass object and delivery was terminted
+			 * We don't fail the fetch, in order for hit-for-pass
+			 * objects to be created.
+			 */
+			AN(bo->fetch_objcore->flags & OC_F_PASS);
+			VSLb(bo->vsl, SLT_FetchError,
+			    "Pass delivery abandonned");
+			vfps = VFP_END;
+			bo->should_close = 1;
+			break;
+		}
 		assert(bo->state != BOS_FAILED);
 		if (st == NULL) {
 			st = VFP_GetStorage(bo, est);
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index dc78078..1419655 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -170,11 +170,13 @@ v1d_WriteDirObj(struct req *req)
 
 	while (1) {
 		ois = ObjIter(oi, &ptr, &len);
-		if (ois == OIS_DATA && !VDP_bytes(req, VDP_NULL,  ptr, len))
-			continue;
-		if (ois == OIS_STREAM && !VDP_bytes(req, VDP_FLUSH,  ptr, len))
-			continue;
-		break;
+		if (ois == OIS_DONE) {
+			AZ(len);
+			break;
+		}
+		if (VDP_bytes(req,
+		     ois == OIS_DATA ? VDP_NULL : VDP_FLUSH,  ptr, len))
+			break;
 	}
 	(void)VDP_bytes(req, VDP_FINISH,  NULL, 0);
 	ObjIterEnd(&oi);
diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index c59ebf9..3c2f670 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -126,8 +126,11 @@ ObjIterEnd(struct objiter **oi)
 	AN(oi);
 	CHECK_OBJ_NOTNULL((*oi), OBJITER_MAGIC);
 	CHECK_OBJ_NOTNULL((*oi)->obj, OBJECT_MAGIC);
-	if ((*oi)->bo != NULL)
+	if ((*oi)->bo != NULL) {
+		if ((*oi)->obj->objcore->flags & OC_F_PASS)
+			(*oi)->bo->abandon = 1;
 		VBO_DerefBusyObj((*oi)->wrk, &(*oi)->bo);
+	}
 	FREE_OBJ((*oi));
 	*oi = NULL;
 }
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index af8531f..adeb37e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -159,9 +159,16 @@ cnt_deliver(struct worker *wrk, struct req *req)
 
 	V1D_Deliver(req);
 
-	/* No point in saving the body if it is hit-for-pass */
-	if (req->obj->objcore->flags & OC_F_PASS)
+	if (req->obj->objcore->flags & OC_F_PASS) {
+		/*
+		 * No point in saving the body if it is hit-for-pass,
+		 * but we can't yank it until the fetching thread has
+		 * finished/abandonned also.
+		 */
+		while (req->obj->objcore->busyobj != NULL)
+			(void)usleep(100000);
 		STV_Freestore(req->obj);
+	}
 
 	assert(WRW_IsReleased(wrk));
 VSLb(req->vsl, SLT_Debug, "XXX REF %d", req->obj->objcore->refcnt);
diff --git a/bin/varnishtest/tests/r01391.vtc b/bin/varnishtest/tests/r01391.vtc
new file mode 100644
index 0000000..4b98fda
--- /dev/null
+++ b/bin/varnishtest/tests/r01391.vtc
@@ -0,0 +1,39 @@
+varnishtest "client abandoning hit-for-pass"
+
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked" -hdr "Set-Cookie: foo=bar"
+	chunked "foo"
+	sema r1 sync 2
+	chunked "bar"
+	delay .1
+	chunkedlen 64
+	delay .1
+	chunkedlen 64
+	chunkedlen 0
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+
+client c1 {
+	txreq
+	rxhdrs
+	rxchunk
+	sema r1 sync 2
+} -run
+
+delay 2
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run



From phk at FreeBSD.org  Tue Dec 17 09:44:31 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Dec 2013 10:44:31 +0100
Subject: [master] 9cd74c3 Add a missing boundary check for Range requests
Message-ID: <E1VsrCZ-0007e3-M8@project.varnish-software.com>


commit 9cd74c3b2cbbe499e98394bb80833dc73f4ec943
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 17 09:43:42 2013 +0000

    Add a missing boundary check for Range requests
    
    Fixes #1323
    Spotted & fix by:	gquintard

diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 1419655..79090fd 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -126,6 +126,8 @@ v1d_dorange(struct req *req, const char *r)
 		}
 		if (!has_low) {
 			low = req->obj->len - high;
+			if (low < 0)
+				low = 0;
 			high = req->obj->len - 1;
 		}
 	} else
diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index faf60fb..a0084d5 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -84,4 +84,9 @@ client c1 {
 	rxresp
 	expect resp.status == 206
 	expect resp.bodylen == 100
+
+	txreq -hdr "Range: bytes=-101" 
+	rxresp 
+	expect resp.status == 206 
+	expect resp.bodylen == 100 
 } -run



From phk at FreeBSD.org  Tue Dec 17 09:50:05 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Dec 2013 10:50:05 +0100
Subject: [master] fc90bf0 This test didn't actually use persistent storage
 because the "shortlived" test puts the object in Transient.
Message-ID: <E1VsrHx-0008HJ-Qg@project.varnish-software.com>


commit fc90bf08720af47673d334053270a69c72e94718
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 17 09:49:21 2013 +0000

    This test didn't actually use persistent storage because the
    "shortlived" test puts the object in Transient.
    
    Patch by:	gquintard
    Fixes		#1387

diff --git a/bin/varnishtest/tests/p00005.vtc b/bin/varnishtest/tests/p00005.vtc
index dc95677..1ac8e08 100644
--- a/bin/varnishtest/tests/p00005.vtc
+++ b/bin/varnishtest/tests/p00005.vtc
@@ -10,15 +10,22 @@ server s1 {
 varnish v1 \
 	-arg "-spersistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0" \
+	-arg "-pshortlived=0" \
 	-vcl+backend {
 		sub vcl_backend_response {
 			set beresp.ttl = 3s;
+			set beresp.keep = 0s;
+			set beresp.grace = 0s;
 		}
 	} -start
 
 varnish v1 -cliok "param.set debug +syncvsl"
 varnish v1 -cliok "param.set feature +wait_silo"
 
+logexpect l1 -v v1 -g vxid -q "Begin ~ bereq" {
+    expect * 1002 Storage         "persistent s0"
+} -start
+
 client c1 {
 	txreq -url "/foo"
 	rxresp
@@ -27,6 +34,7 @@ client c1 {
 	expect resp.http.foo == "foo1"
 } -run
 
+logexpect l1 -wait
 varnish v1 -expect n_object == 1
 
 varnish v1 -stop
@@ -44,6 +52,10 @@ varnish v1 -vcl+backend {
 
 delay 5
 
+logexpect l1 -v v1 -g vxid -q "Begin ~ bereq" {
+    expect * 1002 Storage         "persistent s0"
+} -start
+
 client c1 {
 	txreq -url "/foo"
 	rxresp
@@ -52,4 +64,5 @@ client c1 {
 	expect resp.http.foo == "foo2"
 } -run
 
+logexpect l1 -wait
 varnish v1 -expect n_object == 1



From phk at FreeBSD.org  Tue Dec 17 10:19:19 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Dec 2013 11:19:19 +0100
Subject: [master] d89bbe7 Fix an assert to test that the object has "useful
 lifetime" which is not the same as a positive TTL any more.
Message-ID: <E1VsrkF-0003pK-Gk@project.varnish-software.com>


commit d89bbe7f3123d856751d2d88a945d1401e26de36
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 17 10:17:10 2013 +0000

    Fix an assert to test that the object has "useful lifetime" which
    is not the same as a positive TTL any more.

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 487a77e..d7506f0 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -521,7 +521,7 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo,
 	if (bo->fetch_objcore == NULL)
 		return (NULL);		/* from cnt_error */
 	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
-	AN(bo->exp.ttl > 0.);
+	AN((bo->exp.ttl + bo->exp.grace + bo->exp.keep) > 0.);
 
 	ltot = IRNUP(sc, ltot);
 



From phk at FreeBSD.org  Tue Dec 17 10:19:19 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Dec 2013 11:19:19 +0100
Subject: [master] e8e208b Fix the "shortlived" test to look a the sum of ttl +
 grace + keep rather than just the ttl.
Message-ID: <E1VsrkF-0003pQ-IO@project.varnish-software.com>


commit e8e208b1095c174e264e138ef24d1dd7f19e2215
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 17 10:17:46 2013 +0000

    Fix the "shortlived" test to look a the sum of ttl + grace + keep
    rather than just the ttl.
    
    Fixes		#1268
    Spotted by:	daghf
    Partial fix by:	gquintard

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index eb066eb..16067b2 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -370,7 +370,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->uncacheable)
 		bo->fetch_objcore->flags |= OC_F_PASS;
 
-	if (bo->exp.ttl < cache_param->shortlived || bo->uncacheable == 1)
+	if (bo->uncacheable ||
+	    bo->exp.ttl+bo->exp.grace+bo->exp.keep < cache_param->shortlived)
 		bo->storage_hint = TRANSIENT_STORAGE;
 
 	AZ(bo->stats);
diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c
index e63ce33..ecc36f7 100644
--- a/bin/varnishd/mgt/mgt_param_tbl.c
+++ b/bin/varnishd/mgt/mgt_param_tbl.c
@@ -537,8 +537,8 @@ struct parspec mgt_parspec[] = {
 	{ "shortlived", tweak_timeout,
 		&mgt_param.shortlived,
 		"0", NULL,
-		"Objects created with TTL shorter than this are always "
-		"put in transient storage.",
+		"Objects created with (ttl+grace+keep) shorter than this"
+		" are always put in transient storage.",
 		0,
 		"10.0", "s" },
 	{ "critbit_cooloff", tweak_timeout,



From apj at mutt.dk  Tue Dec 17 11:39:30 2013
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 17 Dec 2013 12:39:30 +0100
Subject: [master] 2852f70 Fix sphinx warning
Message-ID: <E1Vsszq-0003Lu-9G@project.varnish-software.com>


commit 2852f704e5fb8a4186fc42fc2cb5a91e87701fbb
Author: Andreas Plesner <apj at mutt.dk>
Date:   Tue Dec 17 12:28:15 2013 +0100

    Fix sphinx warning

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index a5d1294..ef31f86 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -59,7 +59,7 @@ want to compile Varnish from source for other reasons, follow these
 steps:
 
 We recommend downloading a release tarball, which you can find on
-`repo.varnish-cache.org <http://repo.varnish-cache.org/source/>`_.
+`repo.varnish-cache.org/source <http://repo.varnish-cache.org/source/>`_.
 
 Alternatively, if you want to hack on Varnish, you should clone our
 git repository by doing.



From apj at mutt.dk  Tue Dec 17 11:39:30 2013
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 17 Dec 2013 12:39:30 +0100
Subject: [master] 0b5c383 Start work on a "What's New" section
Message-ID: <E1Vsszq-0003M0-BA@project.varnish-software.com>


commit 0b5c383cd4cc1cbf683c0585f3a7622ad873f9ce
Author: Andreas Plesner <apj at mutt.dk>
Date:   Tue Dec 17 12:36:47 2013 +0100

    Start work on a "What's New" section

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index b3f734b..3d1e28f 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -37,6 +37,7 @@ Contents:
    tutorial/index.rst
    users-guide/index.rst
    reference/index.rst
+   whats-new/index.rst
    phk/index.rst
    glossary/index.rst
 
diff --git a/doc/sphinx/whats-new/changes.rst b/doc/sphinx/whats-new/changes.rst
new file mode 100644
index 0000000..cf027c8
--- /dev/null
+++ b/doc/sphinx/whats-new/changes.rst
@@ -0,0 +1,16 @@
+.. _whatsnew_changes:
+
+Changes in Varnish 4
+====================
+
+Varnish 4 is quite an extensive update over Varnish 3, with some very big improvements to central parts of varnish.
+
+Client/backend split
+--------------------
+In the past, Varnish has fetched the content from the backend in the same
+thread as the client request. The client and backend code has now been split,
+allowing for some much requested improvements.
+This split allows varnish to refresh content in the background while serving
+stale content quickly to the client.
+
+This split has also necessitated a change of the VCL-functions, in particular functionality has moved from the old vcl_fetch method to the two new methods vcl_backend_fetch and vcl_backend_response.
diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst
new file mode 100644
index 0000000..a1d1d73
--- /dev/null
+++ b/doc/sphinx/whats-new/index.rst
@@ -0,0 +1,17 @@
+.. _whats-new-index:
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%
+What's new for Varnish 4.0
+%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+This document describes the changes that have been made for Varnish 4. The
+first section will describe the overarching changes that have gone into
+Varnish, while the second section describes what changes you need to make to
+your configuration as well as any changes in behaviour that you need to take
+into consideration while upgrading.
+
+.. toctree::
+   :maxdepth: 2
+
+   changes
+   upgrading
diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
new file mode 100644
index 0000000..f1fcd99
--- /dev/null
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -0,0 +1,77 @@
+.. _whatsnew_upgrading:
+
+%%%%%%%%%%%%%%%%%%%%%%
+Upgrading to Varnish 4
+%%%%%%%%%%%%%%%%%%%%%%
+
+Changes to VCL
+==============
+
+Much of the VCL syntax has changed in Varnish 4. We've tried to compile a list of changes needed to upgrade here.
+
+Version statement
+~~~~~~~~~~~~~~~~~
+To make sure that people have upgraded their VCL to the current version, varnish now requires the first line of VCL to indicate the VCL version number::
+
+	vcl 4.0;
+
+vcl_fetch is now vcl_backend_response
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Directors have been moved to the vmod_directors
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Use the hash director as a client director
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Since the client director was already a special case of the hash director, it has been removed, and you should use the hash director directly::
+
+	sub vcl_init {
+        	new h = directors.hash();
+        	h.add_backend(b1, 1);
+        	h.add_backend(b2, 1);
+	}
+	
+	sub vcl_recv {
+		set req.backend = h.backend(client.ip);
+	}
+
+error() is now a return value
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+You must now explicitly return an error::
+
+	return(error(999, "Response));
+
+hit_for_pass objects are created using beresp.uncacheable
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Example::
+
+	sub vcl_backend_response {
+		if(beresp.http.X-No-Cache) {
+			set beresp.uncacheable = true;
+			set beresp.ttl = 120s;
+			return(deliver);
+		}
+	}
+
+vcl_recv should return(hash) instead of lookup now
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+req.* not available in vcl_backend_response
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+req.* used to be available in vcl_fetch, but after the split of functionality, you only have bereq.* in vcl_backend_response.
+
+vcl_* reserved
+~~~~~~~~~~~~~~
+Your own subs cannot be named vcl_* anymore. That is reserved for builtin subs.
+
+req.backend.healthy replaced by std.healthy(req.backend)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Changes to parameters
+=====================
+
+linger
+~~~~~~
+
+sess_timeout
+~~~~~~~~~~~~



From apj at mutt.dk  Tue Dec 17 11:39:30 2013
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 17 Dec 2013 12:39:30 +0100
Subject: [master] 627060b Add missing anchor
Message-ID: <E1Vsszq-0003MA-Ch@project.varnish-software.com>


commit 627060b4bc00ef9e46cb5013fe6aca0183937c44
Author: Andreas Plesner <apj at mutt.dk>
Date:   Tue Dec 17 12:39:09 2013 +0100

    Add missing anchor

diff --git a/doc/sphinx/users-guide/vcl-actions.rst b/doc/sphinx/users-guide/vcl-actions.rst
index cfb5019..c981051 100644
--- a/doc/sphinx/users-guide/vcl-actions.rst
+++ b/doc/sphinx/users-guide/vcl-actions.rst
@@ -1,3 +1,5 @@
+.. _user-guide-vcl_actions:
+
 actions
 ~~~~~~~
 



From phk at FreeBSD.org  Tue Dec 17 11:51:26 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Dec 2013 12:51:26 +0100
Subject: [master] 15d6c83 When we get a bogus reply from a backend,
 for instance with too many headers, we need to purge all traces of it,
 before we construct our 503 response,
 otherwise we risk failing to set important headers,
 such as "Connection: close".
Message-ID: <E1VstBO-0006l8-RM@project.varnish-software.com>


commit 15d6c830476bda06c40dbb70ee4bd6be38d6e93e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 17 11:50:13 2013 +0000

    When we get a bogus reply from a backend, for instance with too many
    headers, we need to purge all traces of it, before we construct our
    503 response, otherwise we risk failing to set important headers,
    such as "Connection: close".
    
    Fixes #416
    (again)

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 16067b2..e7ef1bb 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -140,6 +140,22 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 /*--------------------------------------------------------------------
  */
 
+static void
+make_it_503(struct busyobj *bo)
+{
+
+	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, HTTP_Beresp);
+	bo->err_code = 503;
+	http_SetH(bo->beresp, HTTP_HDR_PROTO, "HTTP/1.1");
+	http_SetResp(bo->beresp,
+	    "HTTP/1.1", 503, "Backend fetch failed");
+	http_SetHeader(bo->beresp, "Content-Length: 0");
+	http_SetHeader(bo->beresp, "Connection: close");
+}
+
+/*--------------------------------------------------------------------
+ */
+
 static enum fetch_step
 vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 {
@@ -175,12 +191,7 @@ vbf_stp_fetchhdr(struct worker *wrk, struct busyobj *bo)
 
 	if (i) {
 		AZ(bo->vbc);
-		bo->err_code = 503;
-		http_SetH(bo->beresp, HTTP_HDR_PROTO, "HTTP/1.1");
-		http_SetResp(bo->beresp,
-		    "HTTP/1.1", 503, "Backend fetch failed");
-		http_SetHeader(bo->beresp, "Content-Length: 0");
-		http_SetHeader(bo->beresp, "Connection: close");
+		make_it_503(bo);
 	} else {
 		AN(bo->vbc);
 	}



From phk at FreeBSD.org  Wed Dec 18 11:55:07 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 18 Dec 2013 12:55:07 +0100
Subject: [master] cd0e1da Report sys/usr time for varnishd processes
Message-ID: <E1VtFiV-0007UX-K2@project.varnish-software.com>


commit cd0e1dafff346f1fac73a58d274f839f1594b9f1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Dec 18 11:54:34 2013 +0000

    Report sys/usr time for varnishd processes

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 3348b8e..7eb5cdf 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -31,6 +31,7 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/socket.h>
+#include <sys/resource.h>
 
 #include <fcntl.h>
 #include <inttypes.h>
@@ -571,6 +572,7 @@ varnish_wait(struct varnish *v)
 {
 	void *p;
 	int status, r;
+	struct rusage ru;
 
 	if (v->cli_fd < 0)
 		return;
@@ -585,9 +587,12 @@ varnish_wait(struct varnish *v)
 
 	AZ(pthread_join(v->tp, &p));
 	AZ(close(v->fds[0]));
-	r = wait4(v->pid, &status, 0, NULL);
+	r = wait4(v->pid, &status, 0, &ru);
 	v->pid = 0;
-	vtc_log(v->vl, 2, "R %d Status: %04x", r, status);
+	vtc_log(v->vl, 2, "R %d Status: %04x (u %.6f s %.6f)", r, status,
+	    ru.ru_utime.tv_sec + 1e-6 * ru.ru_utime.tv_usec,
+	    ru.ru_stime.tv_sec + 1e-6 * ru.ru_stime.tv_usec
+	);
 	AZ(pthread_join(v->tp_vsl, &p));
 	if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
 		return;



From phk at FreeBSD.org  Wed Dec 18 12:10:14 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 18 Dec 2013 13:10:14 +0100
Subject: [master] 3136e23 Don't forget to handle OIS_ERROR
Message-ID: <E1VtFx8-0000xg-DR@project.varnish-software.com>


commit 3136e23c611da56ea1537241517d8077667cf4a5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Dec 18 12:09:56 2013 +0000

    Don't forget to handle OIS_ERROR
    
    Hit by:	scoof

diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 79090fd..65c25d5 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -170,16 +170,24 @@ v1d_WriteDirObj(struct req *req)
 	oi = ObjIterBegin(req->wrk, req->obj);
 	XXXAN(oi);
 
-	while (1) {
+	do {
 		ois = ObjIter(oi, &ptr, &len);
-		if (ois == OIS_DONE) {
+		switch(ois) {
+		case OIS_DONE:
 			AZ(len);
 			break;
-		}
-		if (VDP_bytes(req,
-		     ois == OIS_DATA ? VDP_NULL : VDP_FLUSH,  ptr, len))
+		case OIS_ERROR:
 			break;
-	}
+		case OIS_DATA:
+		case OIS_STREAM:
+			if (VDP_bytes(req,
+			     ois == OIS_DATA ? VDP_NULL : VDP_FLUSH,  ptr, len))
+				ois = OIS_ERROR;
+			break;
+		default:
+			WRONG("Wrong OIS value");
+		}
+	} while (ois == OIS_DATA || ois == OIS_STREAM);
 	(void)VDP_bytes(req, VDP_FINISH,  NULL, 0);
 	ObjIterEnd(&oi);
 }



From phk at FreeBSD.org  Fri Dec 20 10:22:17 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 11:22:17 +0100
Subject: [master] 151c79d Add alternate build-system prototype
Message-ID: <E1VtxDl-0007ps-W5@project.varnish-software.com>


commit 151c79d20616fd727b375e52ac678a9b5ef061bd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 09:57:07 2013 +0000

    Add alternate build-system prototype

diff --git a/Makefile.inc.phk b/Makefile.inc.phk
new file mode 100644
index 0000000..24756e0
--- /dev/null
+++ b/Makefile.inc.phk
@@ -0,0 +1,243 @@
+
+.PHONY: default
+default:	all 
+
+WARNS	?=	1
+
+.PHONY: depend all clean install test
+depend all clean install test:
+	@$(MAKE) --no-print-directory -f Makefile.phk TGT=$@ real-$@ \
+	    $(shell env CC=$(CC) WARNS=$(WARNS) sh ${TOPDIR}/config.phk $(TOPDIR) $(CURDIR) )
+
+CFLAGS += $(CF_CFLAGS) $(CF_CWFLAGS)
+CFLAGS += -I$(CURDIR)
+CFLAGS += -I$(TOPDIR)
+CFLAGS += -I$(TOPDIR)/include
+CFLAGS += -I$(TOPDIR)/lib/libvgz
+CFLAGS += -I/usr/local/include
+
+SHLIB_LDFLAGS += -shared -Wl,-x -Wl,--fatal-warnings -Wl,--warn-shared-textrel
+VMOD_LDFLAGS += ${SHLIB_LDFLAGS}
+
+#######################################################################
+
+INSTALL_BASE ?=       $(TOPDIR)/_install
+
+#######################################################################
+
+LIB_VARNISH = -L $(TOPDIR)/lib/libvarnish -lvarnish
+LIB_VARNISHAPI = $(TOPDIR)/lib/libvarnishapi/libvarnishapi.a
+LIB_VARNISHTOOLS = -L $(TOPDIR)/lib/libvarnishtools -lvarnishtools
+LIB_VCC = -L $(TOPDIR)/lib/libvcc -lvcc
+LIB_VGZ = -L $(TOPDIR)/lib/libvgz -lvgz
+LIB_PCRE = -L /usr/local/lib -lpcre
+LIB_EXECINFO = -lexecinfo
+LIB_CURSES = -lcurses
+LIB_READLINE = -lreadline
+
+#######################################################################
+ifdef USE_THREADS
+CFLAGS	+= -pthread -D_THREAD_SAFE -g -O2
+PROG_LDFLAGS += -pthread -D_THREAD_SAFE -g -O2 
+endif
+
+ifdef USE_DLOPEN
+PROG_LDFLAGS += -Wl,-E
+endif
+
+#######################################################################
+ifdef SUBDIRS
+ifeq "$(findstring k,$(MAKEFLAGS))" ""
+	EOPT=set -e
+else
+	EOPT=set +e
+endif
+define run-subdirs
+	$(EOPT) ; for dir in $(SUBDIRS); do \
+		echo "->> $$dir $(@:real-%=%)" ; \
+		$(MAKE) -C $$dir -f Makefile.phk $(@:real-%=%) ; \
+	done
+endef
+else
+define run-subdirs
+endef
+endif
+
+#######################################################################
+ifdef NOWORK
+ifdef SUBDIRS
+
+.PHONY: subdirs $(SUBDIRS)
+
+subdirs: $(SUBDIRS)
+
+$(SUBDIRS):
+	$(MAKE) -C $@ -f Makefile.phk $(@:real-%=%)
+
+TODO += subdirs
+endif
+endif
+
+#######################################################################
+.SUFFIXES: .So
+.c.So:
+	${CC} $(CFLAGS) -fPIC -c $< -o $@
+
+
+#######################################################################
+
+TODO_DEPEND     +=      $(MADE_FILES)
+TODO_ALL        +=      $(MADE_FILES)
+CLEAN_FILES     +=      $(MADE_FILES)
+
+MADE_SRC	=	$(filter %.c, $(MADE_FILES))
+
+#######################################################################
+ifdef PROG_SRC
+PROGNAME = $(notdir $(CURDIR))
+MANNAME=$(PROGNAME).1
+TODO_DEPEND += _.depprog
+TODO_ALL += $(PROGNAME)
+TODO_INSTALL += prog_install
+PROG_SRC += $(MADE_SRC)
+PROG_OBJ = $(notdir $(PROG_SRC:.c=.o))
+CLEAN_FILES += $(PROGNAME) $(PROG_OBJ) _.depprog $(MANNAME)
+
+_.depprog:	${PROG_SRC} ${MADE_FILES}
+	$(CC) $(CFLAGS) -MM $^ > _.depprog
+
+ifeq ($(TGT), all)
+include _.depprog
+endif
+
+$(PROGNAME):	$(PROG_OBJ)
+	$(CC) $(PROG_LDFLAGS) -o $(PROGNAME) $^ $(LD_ADD)
+
+.PHONY: prog_install
+prog_install:	$(PROGNAME)
+	@cp $(PROGNAME) $(INSTALL_BASE)/bin/
+	@[ ! -f $(MANNAME) ] || cp $(MANNAME) $(INSTALL_BASE)/man/
+
+endif
+
+#######################################################################
+ifdef SHLIB_SRC
+SHLIBNAME = $(notdir $(CURDIR))
+TODO_ALL += $(SHLIBNAME).so
+TODO_DEPEND += _.depshlib
+TODO_INSTALL += shlib_install
+SHLIB_SRC += $(MADE_SRC)
+SHLIB_OBJ = $(notdir $(SHLIB_SRC:.c=.So))
+CLEAN_FILES += $(SHLIB_OBJ) $(SHLIBNAME).so _.depshlib
+
+_.depshlib:	${SHLIB_SRC} ${MADE_FILES}
+	$(CC) $(CFLAGS) -MM $^ |sed 's/o:/So:/' > _.depshlib
+
+ifeq ($(TGT), all)
+include _.depshlib
+endif
+
+$(SHLIBNAME).so:	$(SHLIB_OBJ)
+	$(CC) $(SHLIB_LDFLAGS) -o $(SHLIBNAME).so $^
+
+.PHONY: shlib_install
+shlib_install:	$(SHLIBNAME).so
+	@cp $(SHLIBNAME).so $(INSTALL_BASE)/lib/
+
+endif
+
+#######################################################################
+ifdef LIB_SRC
+LIBNAME = $(notdir $(CURDIR))
+TODO_ALL += $(LIBNAME).a
+TODO_DEPEND += _.deplib
+LIB_SRC += $(MADE_SRC)
+LIB_OBJ = $(notdir $(LIB_SRC:.c=.o))
+CLEAN_FILES += $(LIB_OBJ) $(LIBNAME).a _.deplib
+
+_.deplib:	${LIB_SRC} ${MADE_FILES}
+	$(CC) $(CFLAGS) -MM $^ > _.deplib
+
+ifeq ($(TGT), all)
+include _.deplib
+endif
+
+$(LIBNAME).a:	$(LIB_OBJ) 
+	$(AR) -rc $(LIBNAME).a $^
+	ranlib $(LIBNAME).a
+
+endif
+
+#######################################################################
+ifdef VMOD_SRC
+#VMODNAME = $(subst libvmod_,,$(notdir $(CURDIR)))
+VMODNAME = $(notdir $(CURDIR))
+TODO_ALL += vcc_if.h vcc_if.c $(VMODNAME).so
+TODO_DEPEND += _.depvmod
+TODO_INSTALL += vmod_install
+VMOD_SRC += $(MADE_SRC)
+VMOD_SRC += vcc_if.c
+VMOD_OBJ = $(notdir $(VMOD_SRC:.c=.So))
+CLEAN_FILES += $(VMOD_OBJ) vcc_if.c vcc_if.h $(VMODNAME).so _.depvmod
+
+CFLAGS += -I$(TOPDIR)/bin/varnishd
+
+_.depvmod:	${VMOD_SRC} ${MADE_FILES}
+	$(CC) $(CFLAGS) -MM $^ |sed 's/o:/So:/' > _.depvmod
+
+ifeq ($(TGT), all)
+include _.depvmod
+endif
+
+vcc_if.c vcc_if.h: vmod.vcc $(TOPDIR)/lib/libvcc/vmodtool.py
+	${PYTHON} $(TOPDIR)/lib/libvcc/vmodtool.py
+
+$(VMODNAME).so:	$(VMOD_OBJ)
+	$(CC) $(VMOD_LDFLAGS) -o $(VMODNAME).so $^
+
+.PHONY: vmod_install
+vmod_install:	$(VMODNAME).so
+	@cp $(VMODNAME).so $(INSTALL_BASE)/libexec/
+
+endif
+
+#######################################################################
+ifdef RST_MAN
+ifdef HAVE_RST2HTML
+TODO_ALL += ${MANNAME}
+
+${MANNAME}:	${RST_MAN}
+	 rst2man ${RST_MAN} $(MANNAME)
+
+endif
+endif
+
+#######################################################################
+
+.PHONY: real-depend
+real-depend: $(TODO) $(TODO_DEPEND)
+	@$(run-subdirs)
+	@true
+
+.PHONY: real-all
+real-all: $(TODO) $(TODO_ALL)
+	@$(run-subdirs)
+	@true
+
+.PHONY: real-clean
+real-clean: $(TODO) $(TODO_CLEAN)
+	@$(run-subdirs)
+	@[ "x$(CLEAN_FILES)" = "x" ] || rm -f $(CLEAN_FILES)
+
+.PHONY: real-install
+real-install: $(TODO_INSTALL)
+	@$(run-subdirs)
+	@true
+
+.PHONY: real-test
+real-test: $(TODO_TEST)
+	@$(run-subdirs)
+	@true
+
+#######################################################################
+
diff --git a/Makefile.phk b/Makefile.phk
new file mode 100644
index 0000000..237992c
--- /dev/null
+++ b/Makefile.phk
@@ -0,0 +1,15 @@
+SUBDIRS	= lib bin doc
+
+CLEAN_FILES	+= config.h include/vcs_version. include/vmod_abi.h _.cache
+
+TODO_INSTALL +=	top_install
+
+TOPDIR = $(CURDIR)
+include $(TOPDIR)/Makefile.inc.phk
+
+.PHONY: top_install
+top_install:	
+	@for d in man doc etc bin lib libexec ; do \
+		mkdir -p $(INSTALL_BASE)/$$d ; \
+	done
+	
diff --git a/bin/Makefile.phk b/bin/Makefile.phk
new file mode 100644
index 0000000..933c9d8
--- /dev/null
+++ b/bin/Makefile.phk
@@ -0,0 +1,12 @@
+SUBDIRS += varnishadm 
+SUBDIRS += varnishd 
+SUBDIRS += varnishlog
+SUBDIRS += varnishncsa
+# varnishreplay
+SUBDIRS += varnishtest 
+SUBDIRS += varnishstat
+# SUBDIRS += varnishhist 
+# SUBDIRS += varnishtop
+
+TOPDIR = $(CURDIR)/..
+include $(TOPDIR)/Makefile.inc.phk
diff --git a/bin/varnishadm/Makefile.phk b/bin/varnishadm/Makefile.phk
new file mode 100644
index 0000000..6d6cb2a
--- /dev/null
+++ b/bin/varnishadm/Makefile.phk
@@ -0,0 +1,12 @@
+PROG_SRC = varnishadm.c
+
+LD_ADD	+= ${LIB_VARNISHAPI}
+LD_ADD	+= ${LIB_VARNISH}
+LD_ADD	+= ${LIB_PCRE}
+LD_ADD	+= ${LIB_READLINE}
+LD_ADD	+= -lm 
+
+RST_MAN +=  $(TOPDIR)/doc/sphinx/reference/varnishadm.rst
+
+TOPDIR  = $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
diff --git a/bin/varnishd/Makefile.phk b/bin/varnishd/Makefile.phk
new file mode 100644
index 0000000..30c1a89
--- /dev/null
+++ b/bin/varnishd/Makefile.phk
@@ -0,0 +1,121 @@
+VPATH += cache common mgt waiter storage hash
+
+PROG_SRC += cache/cache_acceptor.c
+PROG_SRC += cache/cache_backend.c
+PROG_SRC += cache/cache_backend_cfg.c
+PROG_SRC += cache/cache_backend_poll.c
+PROG_SRC += cache/cache_ban.c
+PROG_SRC += cache/cache_busyobj.c
+PROG_SRC += cache/cache_cli.c
+PROG_SRC += cache/cache_dir.c
+PROG_SRC += cache/cache_esi_deliver.c
+PROG_SRC += cache/cache_esi_fetch.c
+PROG_SRC += cache/cache_esi_parse.c
+PROG_SRC += cache/cache_expire.c
+PROG_SRC += cache/cache_fetch.c
+PROG_SRC += cache/cache_fetch_proc.c
+PROG_SRC += cache/cache_gzip.c
+PROG_SRC += cache/cache_hash.c
+PROG_SRC += cache/cache_http.c
+PROG_SRC += cache/cache_http1_deliver.c
+PROG_SRC += cache/cache_http1_fetch.c
+PROG_SRC += cache/cache_http1_fsm.c
+PROG_SRC += cache/cache_http1_proto.c
+PROG_SRC += cache/cache_lck.c
+PROG_SRC += cache/cache_main.c
+PROG_SRC += cache/cache_mempool.c
+PROG_SRC += cache/cache_obj.c
+PROG_SRC += cache/cache_panic.c
+PROG_SRC += cache/cache_pipe.c
+PROG_SRC += cache/cache_pool.c
+PROG_SRC += cache/cache_req_fsm.c
+PROG_SRC += cache/cache_rfc2616.c
+PROG_SRC += cache/cache_session.c
+PROG_SRC += cache/cache_shmlog.c
+PROG_SRC += cache/cache_vary.c
+PROG_SRC += cache/cache_vcl.c
+PROG_SRC += cache/cache_vrt.c
+PROG_SRC += cache/cache_vrt_re.c
+PROG_SRC += cache/cache_vrt_var.c
+PROG_SRC += cache/cache_vrt_vmod.c
+PROG_SRC += cache/cache_wrk.c
+PROG_SRC += cache/cache_wrw.c
+PROG_SRC += cache/cache_ws.c
+
+PROG_SRC += common/common_vsc.c
+PROG_SRC += common/common_vsm.c
+
+PROG_SRC += hash/hash_classic.c
+PROG_SRC += hash/hash_critbit.c
+PROG_SRC += hash/hash_mgt.c
+PROG_SRC += hash/hash_simple_list.c
+
+PROG_SRC += mgt/mgt_child.c
+PROG_SRC += mgt/mgt_cli.c
+PROG_SRC += mgt/mgt_main.c
+PROG_SRC += mgt/mgt_param.c
+PROG_SRC += mgt/mgt_param_bits.c
+PROG_SRC += mgt/mgt_param_tbl.c
+PROG_SRC += mgt/mgt_param_tweak.c
+PROG_SRC += mgt/mgt_pool.c
+PROG_SRC += mgt/mgt_sandbox.c
+PROG_SRC += mgt/mgt_sandbox_solaris.c
+PROG_SRC += mgt/mgt_shmem.c
+PROG_SRC += mgt/mgt_vcc.c
+
+PROG_SRC += storage/stevedore.c
+PROG_SRC += storage/stevedore_mgt.c
+PROG_SRC += storage/stevedore_utils.c
+PROG_SRC += storage/storage_file.c
+PROG_SRC += storage/storage_malloc.c
+PROG_SRC += storage/storage_persistent.c
+PROG_SRC += storage/storage_persistent_mgt.c
+PROG_SRC += storage/storage_persistent_silo.c
+PROG_SRC += storage/storage_persistent_subr.c
+PROG_SRC += storage/storage_synth.c
+PROG_SRC += storage/storage_umem.c
+
+PROG_SRC += waiter/cache_waiter.c
+PROG_SRC += waiter/cache_waiter_epoll.c
+PROG_SRC += waiter/cache_waiter_kqueue.c
+PROG_SRC += waiter/cache_waiter_poll.c
+PROG_SRC += waiter/cache_waiter_ports.c
+PROG_SRC += waiter/mgt_waiter.c
+
+USE_THREADS = YES
+USE_DLOPEN = YES
+
+LD_ADD	+= ${LIB_VARNISH}
+LD_ADD	+= ${LIB_VCC}
+LD_ADD	+= ${LIB_PCRE}
+LD_ADD	+= ${LIB_EXECINFO}
+LD_ADD	+= ${LIB_VGZ}
+LD_ADD	+= -lm
+
+RST_MAN += $(TOPDIR)/doc/sphinx/reference/varnishd.rst
+
+MADE_FILES += default_vcl.h
+
+TODO_INSTALL += install_etc
+
+TOPDIR  = $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
+#
+# Turn the default.vcl file into a C-string we can include in the program.
+#
+default_vcl.h:	default.vcl
+	echo '/*' > $@
+	echo ' * NB:  This file is machine generated, DO NOT EDIT!' >> $@
+	echo ' *' >> $@
+	echo ' * Edit default.vcl instead and run make' >> $@
+	echo ' *' >> $@
+	echo ' */' >> $@
+	echo '' >> $@
+	sed -e 's/"/\\"/g' \
+	    -e 's/$$/\\n"/' \
+	    -e 's/^/ "/' default.vcl >> $@
+
+.PHONY: install_etc
+install_etc: default_vcl.h
+	@cp default_vcl.h $(INSTALL_BASE)/etc/
diff --git a/bin/varnishhist/Makefile.phk b/bin/varnishhist/Makefile.phk
new file mode 100644
index 0000000..145414d
--- /dev/null
+++ b/bin/varnishhist/Makefile.phk
@@ -0,0 +1,7 @@
+PROG_SRC = varnishhist.c
+
+LD_ADD	+= ${LIB_VARNISH}
+LD_ADD	+= ${LIB_VARNISHAPI}
+
+TOPDIR  = $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
diff --git a/bin/varnishlog/Makefile.phk b/bin/varnishlog/Makefile.phk
new file mode 100644
index 0000000..5f0de52
--- /dev/null
+++ b/bin/varnishlog/Makefile.phk
@@ -0,0 +1,15 @@
+PROG_SRC = varnishlog.c
+PROG_SRC = varnishlog_options.c
+
+LD_ADD	+= ${LIB_VARNISHTOOLS}
+LD_ADD	+= ${LIB_VARNISHAPI}
+LD_ADD	+= ${LIB_VARNISH}
+LD_ADD	+= ${LIB_PCRE}
+LD_ADD	+= -lm
+
+TODO_ALL += build_man
+
+TOPDIR  = $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
+.PHONY: build_man
diff --git a/bin/varnishncsa/Makefile.phk b/bin/varnishncsa/Makefile.phk
new file mode 100644
index 0000000..10fd638
--- /dev/null
+++ b/bin/varnishncsa/Makefile.phk
@@ -0,0 +1,16 @@
+PROG_SRC += base64.c
+PROG_SRC += varnishncsa.c
+PROG_SRC += varnishncsa_options.c
+
+LD_ADD	+= ${LIB_VARNISHTOOLS}
+LD_ADD	+= ${LIB_VARNISHAPI}
+LD_ADD	+= ${LIB_VARNISH}
+LD_ADD	+= ${LIB_PCRE}
+LD_ADD	+= -lm
+
+TODO_ALL += build_man
+
+TOPDIR  = $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
+.PHONY: build_man
diff --git a/bin/varnishreplay/Makefile.phk b/bin/varnishreplay/Makefile.phk
new file mode 100644
index 0000000..d69f03e
--- /dev/null
+++ b/bin/varnishreplay/Makefile.phk
@@ -0,0 +1,7 @@
+PROG_SRC = varnishreplay.c
+
+LD_ADD	+= ${LIB_VARNISH}
+LD_ADD	+= ${LIB_VARNISHAPI}
+
+TOPDIR  = $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
diff --git a/bin/varnishstat/Makefile.phk b/bin/varnishstat/Makefile.phk
new file mode 100644
index 0000000..3a89de7
--- /dev/null
+++ b/bin/varnishstat/Makefile.phk
@@ -0,0 +1,13 @@
+PROG_SRC += varnishstat.c
+PROG_SRC += varnishstat_curses.c
+
+LD_ADD	+= ${LIB_VARNISHAPI}
+LD_ADD	+= ${LIB_VARNISH}
+LD_ADD	+= ${LIB_PCRE}
+LD_ADD	+= ${LIB_CURSES}
+LD_ADD	+= -lm
+
+RST_MAN += $(TOPDIR)/doc/sphinx/reference/varnishstat.rst
+
+TOPDIR  = $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
diff --git a/bin/varnishtest/Makefile.phk b/bin/varnishtest/Makefile.phk
new file mode 100644
index 0000000..748977e
--- /dev/null
+++ b/bin/varnishtest/Makefile.phk
@@ -0,0 +1,29 @@
+PROG_SRC += vtc.c
+PROG_SRC += vtc_client.c
+PROG_SRC += vtc_http.c
+PROG_SRC += vtc_log.c
+PROG_SRC += vtc_logexp.c
+PROG_SRC += vtc_main.c
+PROG_SRC += vtc_sema.c
+PROG_SRC += vtc_server.c
+PROG_SRC += vtc_varnish.c
+
+LD_ADD	+= ${LIB_VARNISHAPI}
+LD_ADD	+= ${LIB_VARNISH}
+LD_ADD	+= ${LIB_PCRE}
+LD_ADD	+= ${LIB_VGZ}
+LD_ADD	+= -lm
+
+USE_THREADS = yes
+
+RST_MAN += $(TOPDIR)/doc/sphinx/reference/varnishtest.rst
+
+TODO_TEST	+=	all_tests
+
+TOPDIR  = $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
+.PHONY:	all_tests
+
+all_tests:
+	./varnishtest -i -j 4 tests/*.vtc
diff --git a/bin/varnishtop/Makefile.phk b/bin/varnishtop/Makefile.phk
new file mode 100644
index 0000000..1b922f3
--- /dev/null
+++ b/bin/varnishtop/Makefile.phk
@@ -0,0 +1,9 @@
+PROG_SRC = varnishtop.c
+
+LD_ADD	+= ${LIB_VARNISH}
+LD_ADD	+= ${LIB_VARNISHAPI}
+LD_ADD	+= ${LIB_PCRE}
+LD_ADD	+= ${LIB_CURSES}
+
+TOPDIR  = $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
diff --git a/config.phk b/config.phk
new file mode 100644
index 0000000..89ed5f9
--- /dev/null
+++ b/config.phk
@@ -0,0 +1,249 @@
+#!/bin/sh
+
+set -e
+
+#######################################################################
+# Adminstrative settings
+
+ADM_PROJECT=varnish
+ADM_VERSION=trunk
+
+CONFIG_H=phk_hack/config.h
+
+#######################################################################
+# We always rebuild the config when make is run in the toplevel directory
+# Otherwise, we try to use a cached config, if we have one.
+#
+# We know we're in the toplevel directory, because we get passed
+# TOPDIR and CURDIR as arguments
+
+if [ "x$1" = "x" -o "x$2" = "x" ] ; then
+	echo "Missing arguments, assuming topdir" 1>&2
+	set `pwd` `pwd`
+fi
+
+if [ "x$1" = "x$2" ] ; then
+	rm -f _.cache
+elif [ "x$1" != "x" -a -d $1 ] ; then
+	cd $1
+fi
+
+#######################################################################
+# autocrap co-existence
+# We put our config.h somewhere else and delete autocraps.
+# Autocrap regenerates its own config.h and doesn't know about ours
+
+mkdir -p phk_hack
+rm -f config.h
+
+#######################################################################
+
+
+if [ -f _.cache -a -f ${CONFIG_H} ] ; then
+	cat _.cache
+	exit 0
+fi
+
+echo "Building Config" 1>&2
+
+#######################################################################
+# Look for #include files for HAVE_ etc.
+
+if true ; then
+	rm -f ${CONFIG_H}_
+	echo '' > ${CONFIG_H}_
+
+	for i in \
+		curses.h \
+		endian.h \
+		execinfo.h \
+		ncurses.h \
+		ncurses/curses.h \
+		ncursesw.h \
+		ncursesw/curses.h \
+		priv.h \
+		pthread_np.h \
+		readline/history.h \
+		readline/readline.h \
+		sys/endian.h \
+		sys/filio.h \
+		sys/mount.h \
+		sys/statvfs.h \
+		sys/vfs.h \
+		umem.h 
+	do
+		if [ -f /usr/include/$i ] ; then
+			n=`echo $i | tr '[a-z/.]' '[A-Z__]'`
+			echo "#define HAVE_$n 1" >> ${CONFIG_H}_
+		fi
+	done
+	echo "#define PACKAGE_TARNAME \"${ADM_PROJECT}\"" >> ${CONFIG_H}_
+	echo "#define PACKAGE_VERSION \"${ADM_VERSION}\"" >> ${CONFIG_H}_
+
+	echo '
+		#define VARNISH_STATE_DIR       "/tmp/phk/"
+	' >> ${CONFIG_H}_
+	if [ ! -f ${CONFIG_H} ] ; then
+		mv ${CONFIG_H}_ ${CONFIG_H}
+	elif ! cmp -s ${CONFIG_H} ${CONFIG_H}_ ; then
+		mv ${CONFIG_H}_ ${CONFIG_H}
+	else
+		rm -f ${CONFIG_H}_
+	fi
+fi
+
+#######################################################################
+# Create files depending on VCS (git) output
+
+VCSF=include/vcs_version.h
+VMAV=include/vmod_abi.h
+
+if [ -d ./.git ] ; then 
+	V=`git show -s --pretty=format:%h`
+else
+	V="NOGIT"
+fi
+(
+echo "/* $V */"
+echo "/*"
+echo " * NB:  This file is machine generated, DO NOT EDIT!"
+echo " *"
+echo " * make(1) updates this when necessary"
+echo " *"
+echo " */"
+echo "#define VCS_Version \"$V\""
+) > ${VCSF}_
+if [ ! -f ${VCSF} ] ; then
+	mv ${VCSF}_ ${VCSF}
+	rm -f ${VMAV}
+elif ! cmp -s ${VCSF}_ ${VCSF} ; then
+	mv ${VCSF}_ ${VCSF}
+	rm -f ${VMAV}
+else
+	rm ${VCSF}_
+fi
+
+if [ ! -f ${VMAV} ] ; then
+	echo "#define VMOD_ABI_Version \"Varnish trunk $V\"" > ${VMAV}
+fi
+
+#######################################################################
+# Ask the compiler about stuff
+
+ask_compiler() {
+	a=`echo "$2" |
+	    ${CC-cc} -E - -DWARNS=${WARNS-1} |
+	    sed -e '/^#/d' -e '/^$/d' -e 's/"//g' -e 's/~/"/g' ` 
+	echo "$1='" $a "'" >> _.cache
+}
+
+# Warning flags
+ask_compiler CF_CWFLAGS '
+	#if WARNS >= 1
+		"-Wall"
+		"-Werror"
+	#endif
+	#if WARNS >= 2
+		"-W"
+		"-fstack-protector"
+		"-Wno-format-y2k"
+		"-Wno-unused-parameter"
+		"-Wstrict-prototypes"
+		"-Wmissing-prototypes"
+		"-Wpointer-arith"
+		"-Wreturn-type"
+		"-Wcast-qual"
+		"-Wwrite-strings"
+		"-Wswitch"
+		"-Wshadow"
+		"-Wunused-parameter"
+		"-Wcast-align"
+		"-Wchar-subscripts"
+		"-Winline"
+		"-Wnested-externs"
+		"-Wno-pointer-sign"
+		"-Wno-empty-body"
+		"-Wextra"
+		"-Wno-missing-field-initializers"
+		"-Wno-sign-compare"
+		#if defined(__clang__)
+			"-Wmissing-variable-declarations"
+			"-Wno-string-plus-int"
+		#endif
+	#endif
+	'
+
+# Configuration options
+ask_compiler CF_CFLAGS '
+	#if defined(__SVR4) && defined(sun)
+		// Solaris and OmniOS
+		"-DHAVE_GETHRTIME"
+		"-DHAVE_PORT_CREATE"
+		"-DHAVE_SETPPRIV"
+	#endif
+
+	#if !defined(__APPLE__)
+		"-DHAVE_DAEMON"
+	#endif
+
+	// Where does this not work ?
+		"-DSO_SNDTIMEO_WORKS"
+		"-DSO_RCVTIMEO_WORKS"
+		"-DHAVE_TCP_KEEP"
+
+	"-DVCC_CC=~\~exec $(CC) -D_THREAD_SAFE -std=gnu99 -g -O2 -Wall -Werror -pthread -fpic -shared -Wl,-x -o %o %s\~~"
+
+	"-DVCC_WARNS=~\~$(CF_CWFLAGS)\~~"
+
+	"-I$(TOPDIR)/phk_hack"
+
+	'
+
+#######################################################################
+# Find a Python interpreter
+#
+
+for i in 3.2 2.7 "" 2.5 2.6 3.0 3.1
+do
+	if python$i < /dev/null > /dev/null 2>&1 ; then
+		echo PYTHON=python$i >> _.cache
+		break
+	fi
+done
+
+#######################################################################
+# Find a rst2* tools
+#
+
+if echo | rst2html > /dev/null 2>&1 ; then
+	echo "HAVE_RST2HTML=1" >> _.cache
+fi
+
+cat _.cache
+exit 0
+
+
+# HAVE_ACCEPT_FILTERS +
+# HAVE_BACKTRACE +
+# HAVE_CLOCK_GETTIME +
+# HAVE_DAEMON +
+# HAVE_DLADDR +
+# HAVE_KQUEUE +
+# HAVE_NANOSLEEP +
+# HAVE_PTHREAD_SET_NAME_NP +
+# HAVE_SETPROCTITLE +
+# HAVE_SRANDOMDEV +
+# HAVE_TCP_KEEP +
+# HAVE_TIMEGM +
+# PACKAGE_TARNAME +
+# PACKAGE_VERSION +
+# SO_RCVTIMEO_WORKS +
+# SO_SNDTIMEO_WORKS +
+# VCC_CC +
+# HAVE_EPOLL_CTL -
+# HAVE_LIBUMEM -
+# USE_PCRE_JIT -
+# _FILE_OFFSET_BITS -
+# 
+# #echo 'CF_CFLAGS="-Wall"'
+
diff --git a/doc/Makefile.phk b/doc/Makefile.phk
new file mode 100644
index 0000000..fe6059e
--- /dev/null
+++ b/doc/Makefile.phk
@@ -0,0 +1,7 @@
+
+ifdef HAVE_RST2HTML
+SUBDIRS	+=	sphinx
+endif
+
+TOPDIR = ..
+include $(TOPDIR)/Makefile.inc.phk
diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
new file mode 100644
index 0000000..b14e6cb
--- /dev/null
+++ b/doc/sphinx/Makefile.phk
@@ -0,0 +1,190 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = =build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(CURDIR)
+
+.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest
+
+all:	conf.py html
+
+conf.py: conf.py.in
+	cp conf.py.in conf.py
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html      to make standalone HTML files"
+	@echo "  dirhtml   to make HTML files named index.html in directories"
+	@echo "  pickle    to make pickle files"
+	@echo "  json      to make JSON files"
+	@echo "  htmlhelp  to make HTML files and a HTML help project"
+	@echo "  qthelp    to make HTML files and a qthelp project"
+	@echo "  latex     to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  changes   to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck to check all external links for integrity"
+	@echo "  doctest   to run all doctests embedded in the documentation (if enabled)"
+
+.PHONY: depend install test
+depend install test:
+	@true
+
+clean:
+	-rm -rf $(BUILDDIR)/* conf.py
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Varnish.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Varnish.qhc"
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
+	      "run these through (pdf)latex."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+EXTRA_DIST = \
+	conf.py \
+	index.rst \
+	glossary/index.rst \
+	installation/bugs.rst \
+	installation/help.rst \
+	installation/index.rst \
+	installation/install.rst \
+	installation/prerequisites.rst \
+	installation/platformnotes.rst \
+	phk/autocrap.rst \
+	phk/backends.rst \
+	phk/barriers.rst \
+	phk/gzip.rst \
+	phk/index.rst \
+	phk/ipv6suckage.rst \
+	phk/platforms.rst \
+	phk/sphinx.rst \
+	phk/ssl.rst \
+	phk/thoughts.rst \
+	phk/three-zero.rst \
+	phk/vcl_expr.rst \
+	phk/wanton_destruction.rst \
+	reference/index.rst \
+	reference/params.rst \
+	reference/varnishadm.rst \
+	reference/varnish-cli.rst \
+	reference/varnishd.rst \
+	reference/varnishhist.rst \
+	reference/varnishlog.rst \
+	reference/varnishncsa.rst \
+	reference/varnishreplay.rst \
+	reference/varnishsizes.rst \
+	reference/varnishstat.rst \
+	reference/varnishtest.rst \
+	reference/varnishtop.rst \
+	reference/vcl.rst \
+	reference/vmod.rst \
+	reference/vmod_std.rst \
+	reference/vsm.rst \
+	reference/vsl-query.rst \
+	reference/vsl.rst \
+	tutorial/index.rst \
+	tutorial/introduction.rst \
+	tutorial/starting_varnish.rst \
+	tutorial/putting_varnish_on_port_80.rst \
+	tutorial/backend_servers.rst \
+	tutorial/now_what.rst \
+	users-guide/command-line.rst \
+	users-guide/compression.rst \
+	users-guide/cookies.rst \
+	users-guide/devicedetection.rst \
+	users-guide/esi.rst \
+	users-guide/increasing-your-hitrate.rst \
+	users-guide/index.rst \
+	users-guide/intro.rst \
+	users-guide/operation-cli.rst \
+	users-guide/operation-logging.rst \
+	users-guide/operation.rst \
+	users-guide/operation-statistics.rst \
+	users-guide/params.rst \
+	users-guide/performance.rst \
+	users-guide/purging.rst \
+	users-guide/report.rst \
+	users-guide/running.rst \
+	users-guide/sizing-your-cache.rst \
+	users-guide/storage-backends.rst \
+	users-guide/troubleshooting.rst \
+	users-guide/vary.rst \
+	users-guide/vcl-actions.rst \
+	users-guide/vcl-backends.rst \
+	users-guide/vcl-examples.rst \
+	users-guide/vcl-hashing.rst \
+	users-guide/vcl-inline-c.rst \
+	users-guide/vcl.rst \
+	users-guide/vcl-saint-and-grace.rst \
+	users-guide/vcl-syntax.rst \
+	users-guide/vcl-variables.rst \
+	users-guide/websockets.rst
+
+
+dist-hook:
+	$(MAKE) html
+	cp -r $(BUILDDIR) $(distdir)
+
+distclean-local:
+	rm -rf $(BUILDDIR)
+
+# XXX: doesn't work...
+#reference/params.rst: $(top_builddir)/bin/varnishd/varnishd
+#	$(top_builddir)/bin/varnishd/varnishd -x dumprstparam > reference/params.rst
diff --git a/lib/Makefile.phk b/lib/Makefile.phk
new file mode 100644
index 0000000..6eba2c8
--- /dev/null
+++ b/lib/Makefile.phk
@@ -0,0 +1,6 @@
+SUBDIRS = libvcc libvarnish libvarnishapi libvarnishtools libvcc
+SUBDIRS += libvgz 
+SUBDIRS += libvmod_debug libvmod_directors libvmod_std
+
+TOPDIR	= $(CURDIR)/..
+include $(TOPDIR)/Makefile.inc.phk
diff --git a/lib/libvarnish/Makefile.phk b/lib/libvarnish/Makefile.phk
new file mode 100644
index 0000000..53f4cfe
--- /dev/null
+++ b/lib/libvarnish/Makefile.phk
@@ -0,0 +1,29 @@
+
+LIB_SRC += binary_heap.c
+LIB_SRC += cli_auth.c
+LIB_SRC += cli_common.c
+LIB_SRC += cli_serve.c
+LIB_SRC += flopen.c
+LIB_SRC += vas.c
+LIB_SRC += vav.c
+LIB_SRC += vct.c
+LIB_SRC += version.c
+LIB_SRC += vev.c
+LIB_SRC += vfil.c
+LIB_SRC += vin.c
+LIB_SRC += vlu.c
+LIB_SRC += vmb.c
+LIB_SRC += vnum.c
+LIB_SRC += vpf.c
+LIB_SRC += vre.c
+LIB_SRC += vsa.c
+LIB_SRC += vsb.c
+LIB_SRC += vsha256.c
+LIB_SRC += vss.c
+LIB_SRC += vsub.c
+LIB_SRC += vtcp.c
+LIB_SRC += vtim.c
+
+TOPDIR= $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
diff --git a/lib/libvarnishapi/Makefile.phk b/lib/libvarnishapi/Makefile.phk
new file mode 100644
index 0000000..8e86df4
--- /dev/null
+++ b/lib/libvarnishapi/Makefile.phk
@@ -0,0 +1,37 @@
+
+LIB_SRC += vsc.c
+LIB_SRC += vsl.c
+LIB_SRC += vsl2rst.c
+LIB_SRC += vsl_arg.c
+LIB_SRC += vsl_cursor.c
+LIB_SRC += vsl_dispatch.c
+LIB_SRC += vsl_query.c
+LIB_SRC += vsm.c
+LIB_SRC += vxp.c
+LIB_SRC += vxp_fixed_token.c
+LIB_SRC += vxp_lexer.c
+LIB_SRC += vxp_parse.c
+
+
+# We add more stuff to the SHLIB version to make it self-contained
+SHLIB_SRC = $(LIB_SRC)
+VPATH += ../libvarnish
+SHLIB_SRC += vre.c
+SHLIB_SRC += vsb.c
+SHLIB_SRC += vas.c
+SHLIB_SRC += vav.c
+SHLIB_SRC += vin.c
+SHLIB_SRC += vtim.c
+
+#LIB_SRC += vsl_glob_test.c
+#LIB_SRC += vxp_test.c
+
+MADE_FILES += vxp_fixed_token.c vxp_tokens.h
+
+TOPDIR= $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
+$(MADE_FILES):  generate.py
+	python generate.py
+
+
diff --git a/lib/libvarnishtools/Makefile.phk b/lib/libvarnishtools/Makefile.phk
new file mode 100644
index 0000000..366a04d
--- /dev/null
+++ b/lib/libvarnishtools/Makefile.phk
@@ -0,0 +1,7 @@
+
+LIB_SRC += opt2rst.c
+LIB_SRC += vut.c
+
+TOPDIR= $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
diff --git a/lib/libvcc/Makefile.phk b/lib/libvcc/Makefile.phk
new file mode 100644
index 0000000..131f850
--- /dev/null
+++ b/lib/libvcc/Makefile.phk
@@ -0,0 +1,26 @@
+
+LIB_SRC += vcc_acl.c
+LIB_SRC += vcc_action.c
+LIB_SRC += vcc_backend.c
+LIB_SRC += vcc_backend_util.c
+LIB_SRC += vcc_compile.c
+LIB_SRC += vcc_expr.c
+LIB_SRC += vcc_parse.c
+LIB_SRC += vcc_storage.c
+LIB_SRC += vcc_symb.c
+LIB_SRC += vcc_token.c
+LIB_SRC += vcc_utils.c
+LIB_SRC += vcc_var.c
+LIB_SRC += vcc_vmod.c
+LIB_SRC += vcc_xref.c
+
+MADE_FILES += vcc_fixed_token.c vcc_obj.c vcc_token_defs.h
+
+CLEAN_FILES += $(MADE_FILES)
+
+TOPDIR= $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
+$(MADE_FILES):	generate.py
+	$(PYTHON) generate.py
+
diff --git a/lib/libvgz/Makefile.phk b/lib/libvgz/Makefile.phk
new file mode 100644
index 0000000..b938312
--- /dev/null
+++ b/lib/libvgz/Makefile.phk
@@ -0,0 +1,17 @@
+LIB_SRC += adler32.c
+LIB_SRC += compress.c
+LIB_SRC += crc32.c
+LIB_SRC += deflate.c
+LIB_SRC += infback.c
+LIB_SRC += inffast.c
+LIB_SRC += inflate.c
+LIB_SRC += inftrees.c
+LIB_SRC += trees.c
+LIB_SRC += uncompr.c
+LIB_SRC += zutil.c
+
+CFLAGS += -DZLIB_CONST -D_LARGEFILE64_SOURCE=1
+
+TOPDIR= $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
diff --git a/lib/libvmod_debug/Makefile.phk b/lib/libvmod_debug/Makefile.phk
new file mode 100644
index 0000000..98e52e7
--- /dev/null
+++ b/lib/libvmod_debug/Makefile.phk
@@ -0,0 +1,7 @@
+
+VMOD_SRC += vmod_debug.c
+VMOD_SRC += vmod_debug_obj.c
+
+TOPDIR= $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
diff --git a/lib/libvmod_directors/Makefile.phk b/lib/libvmod_directors/Makefile.phk
new file mode 100644
index 0000000..424a275
--- /dev/null
+++ b/lib/libvmod_directors/Makefile.phk
@@ -0,0 +1,10 @@
+
+VMOD_SRC += fall_back.c
+VMOD_SRC += hash.c
+VMOD_SRC += random.c
+VMOD_SRC += round_robin.c
+VMOD_SRC += vdir.c
+
+TOPDIR= $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+
diff --git a/lib/libvmod_std/Makefile.phk b/lib/libvmod_std/Makefile.phk
new file mode 100644
index 0000000..ca093a0
--- /dev/null
+++ b/lib/libvmod_std/Makefile.phk
@@ -0,0 +1,8 @@
+
+VMOD_SRC += vmod_std.c
+VMOD_SRC += vmod_std_conversions.c
+VMOD_SRC += vmod_std_fileread.c
+
+TOPDIR= $(CURDIR)/../..
+include $(TOPDIR)/Makefile.inc.phk
+



From phk at FreeBSD.org  Fri Dec 20 11:54:12 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 12:54:12 +0100
Subject: [master] 4c182b5 Remove srandomdev() compat include,
 these files don't use srandomdev() any more
Message-ID: <E1Vtyei-0001jZ-Uq@project.varnish-software.com>


commit 4c182b5c66d3d6872b250d572a75818c84391a4f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 11:53:49 2013 +0000

    Remove srandomdev() compat include, these files don't use
    srandomdev() any more

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index adeb37e..d80c16f 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -71,10 +71,6 @@ DOT acceptor -> recv [style=bold,color=green]
 #include "vsha256.h"
 #include "vtim.h"
 
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
-
 /*--------------------------------------------------------------------
  * Deliver an already stored object
  *
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index bedd463..470c931 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -45,11 +45,6 @@
 #include "vav.h"
 #include "vtim.h"
 
-
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
-
 #define		MAX_TOKENS		200
 
 static char		*vtc_desc;



From phk at FreeBSD.org  Fri Dec 20 12:19:06 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 13:19:06 +0100
Subject: [master] 4f38c11 Roll our on VRND_Seed() and us that all the time,
 rather than bother with compat-crap for srandomdev()
Message-ID: <E1Vtz2o-0004BI-FS@project.varnish-software.com>


commit 4f38c1169b0edbe7c18d6fa207a2da5da8a7f752
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 12:18:32 2013 +0000

    Roll our on VRND_Seed() and us that all the time, rather than
    bother with compat-crap for srandomdev()

diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 2749338..8e116c9 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -36,13 +36,11 @@
 #include "common/heritage.h"
 
 #include "vcli_priv.h"
+#include "vrnd.h"
 
 #include "waiter/waiter.h"
 #include "hash/hash_slinger.h"
 
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
 
 volatile struct params	*cache_param;
 
@@ -236,7 +234,7 @@ child_main(void)
 
 	BAN_Compile();
 
-	srandomdev();
+	VRND_Seed();
 	srand48(random());
 	CLI_AddFuncs(debug_cmds);
 
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index e19241f..f977adb 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -52,15 +52,12 @@
 #include "vcli_serve.h"
 #include "vev.h"
 #include "vlu.h"
+#include "vrnd.h"
 #include "vss.h"
 #include "vtcp.h"
 
 #include "mgt_cli.h"
 
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
-
 static int		cli_i = -1, cli_o = -1;
 static struct VCLS	*cls;
 static const char	*secret_file;
@@ -259,7 +256,7 @@ mgt_cli_challenge(struct cli *cli)
 {
 	int i;
 
-	srandomdev();
+	VRND_Seed();
 	for (i = 0; i + 2L < sizeof cli->challenge; i++)
 		cli->challenge[i] = (random() % 26) + 'a';
 	cli->challenge[i++] = '\n';
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index cdd7ca6..cd007b1 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -56,15 +56,12 @@
 #include "vfil.h"
 #include "vin.h"
 #include "vpf.h"
+#include "vrnd.h"
 #include "vsha256.h"
 #include "vtim.h"
 
 #include "compat/daemon.h"
 
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
-
 struct heritage		heritage;
 unsigned		d_flag = 0;
 pid_t			mgt_pid;
@@ -332,7 +329,7 @@ make_secret(const char *dirname)
 		    dirname, strerror(errno));
 		exit(1);
 	}
-	srandomdev();
+	VRND_Seed();
 	for (i = 0; i < sizeof buf; i++)
 		buf[i] = random() & 0xff;
 	assert(sizeof buf == write(fd, buf, sizeof buf));
@@ -413,7 +410,7 @@ main(int argc, char * const *argv)
 	for (o = getdtablesize(); o > STDERR_FILENO; o--)
 		(void)close(o);
 
-	srandomdev();
+	VRND_Seed();
 
 	mgt_got_fd(STDERR_FILENO);
 
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index d6169e6..9e2d0c8 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -44,12 +44,9 @@
 
 #include "vev.h"
 #include "vqueue.h"
+#include "vrnd.h"
 #include "vtim.h"
 
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
-
 #define		MAX_FILESIZE		(1024 * 1024)
 
 
@@ -270,7 +267,7 @@ start_test(void)
 	assert(jp->buf != MAP_FAILED);
 	memset(jp->buf, 0, jp->bufsiz);
 
-	srandomdev();
+	VRND_Seed();
 	bprintf(tmpdir, "%s/vtc.%d.%08x", tmppath, (int)getpid(),
 		(unsigned)random());
 	AZ(mkdir(tmpdir, 0711));
diff --git a/configure.ac b/configure.ac
index cd346b3..ea2039a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -257,7 +257,6 @@ AC_SUBST(LIBUMEM)
 # These functions are provided by libcompat on platforms where they
 # are not available
 AC_CHECK_FUNCS([setproctitle])
-AC_CHECK_FUNCS([srandomdev])
 AC_SEARCH_LIBS(backtrace, [execinfo],[AC_DEFINE([HAVE_BACKTRACE],[1],[Define to 1 if the backtrace function exists])])
 # white lie - we don't actually test it
 AC_MSG_CHECKING([whether daemon() works])
diff --git a/include/Makefile.am b/include/Makefile.am
index 5b75aca..17633ce 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -40,7 +40,6 @@ nobase_noinst_HEADERS = \
 	binary_heap.h \
 	compat/daemon.h \
 	compat/execinfo.h \
-	compat/srandomdev.h \
 	flopen.h \
 	libvcc.h \
 	persistent.h \
@@ -57,6 +56,7 @@ nobase_noinst_HEADERS = \
 	vmb.h \
 	vnum.h \
 	vpf.h \
+	vrnd.h \
 	vsub.h \
 	vss.h \
 	vtcp.h \
diff --git a/include/compat/srandomdev.h b/include/compat/srandomdev.h
deleted file mode 100644
index e1b8c78..0000000
--- a/include/compat/srandomdev.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Dag-Erling Sm?rgrav <des at des.no>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#ifndef COMPAT_SRANDOMDEV_H_INCLUDED
-#define COMPAT_SRANDOMDEV_H_INCLUDED
-
-#ifndef HAVE_SRANDOMDEV
-void srandomdev(void);
-#endif
-
-#endif
diff --git a/include/vrnd.h b/include/vrnd.h
new file mode 100644
index 0000000..ac46c0d
--- /dev/null
+++ b/include/vrnd.h
@@ -0,0 +1,32 @@
+/*-
+ * Copyright (c) 2013 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Random functions
+ */
+
+void VRND_Seed(void);		/* Seed random(3) properly */
+
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index c471c9c..4e09fb9 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -27,6 +27,7 @@ libvarnish_la_SOURCES = \
 	vmb.c \
 	vpf.c \
 	vre.c \
+	vrnd.c \
 	vsa.c \
 	vsb.c \
 	vsha256.c \
diff --git a/lib/libvarnish/Makefile.phk b/lib/libvarnish/Makefile.phk
index 53f4cfe..677e687 100644
--- a/lib/libvarnish/Makefile.phk
+++ b/lib/libvarnish/Makefile.phk
@@ -16,6 +16,7 @@ LIB_SRC += vmb.c
 LIB_SRC += vnum.c
 LIB_SRC += vpf.c
 LIB_SRC += vre.c
+LIB_SRC += vrnd.c
 LIB_SRC += vsa.c
 LIB_SRC += vsb.c
 LIB_SRC += vsha256.c
diff --git a/lib/libvarnish/vrnd.c b/lib/libvarnish/vrnd.c
new file mode 100644
index 0000000..655cf7e
--- /dev/null
+++ b/lib/libvarnish/vrnd.c
@@ -0,0 +1,78 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dag-Erling Sm?rgrav <des at des.no>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "vas.h"
+#include "vrnd.h"
+#include "vtim.h"
+#include "vsha256.h"
+
+void
+VRND_Seed(void)
+{
+	unsigned long seed;
+	struct SHA256Context ctx;
+	double d;
+	pid_t p;
+	unsigned char b[SHA256_LEN];
+	int fd;
+	ssize_t sz;
+
+	fd = open("/dev/urandom", O_RDONLY);
+	if (fd < 0)
+		fd = open("/dev/random", O_RDONLY);
+	if (fd >= 0) {
+		sz = read(fd, &seed, sizeof seed);
+		AZ(close(fd));
+		if (sz == sizeof seed) {
+			srandom(seed);
+			return;
+		}
+	}
+
+	SHA256_Init(&ctx);
+	d = VTIM_mono();
+	SHA256_Update(&ctx, &d, sizeof d);
+	d = VTIM_real();
+	SHA256_Update(&ctx, &d, sizeof d);
+	p = getpid();
+	SHA256_Update(&ctx, &p, sizeof p);
+	p = getppid();
+	SHA256_Update(&ctx, &p, sizeof p);
+	SHA256_Final(b, &ctx);
+	memcpy(&seed, b, sizeof seed);
+	srandom(seed);
+}
diff --git a/lib/libvarnishcompat/Makefile.am b/lib/libvarnishcompat/Makefile.am
index 698531f..543fbb2 100644
--- a/lib/libvarnishcompat/Makefile.am
+++ b/lib/libvarnishcompat/Makefile.am
@@ -9,5 +9,4 @@ libvarnishcompat_la_LDFLAGS = $(AM_LDFLAGS) -avoid-version
 
 libvarnishcompat_la_SOURCES = \
 	daemon.c \
-	execinfo.c \
-	srandomdev.c
+	execinfo.c
diff --git a/lib/libvarnishcompat/srandomdev.c b/lib/libvarnishcompat/srandomdev.c
deleted file mode 100644
index 83b22c4..0000000
--- a/lib/libvarnishcompat/srandomdev.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Dag-Erling Sm?rgrav <des at des.no>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#ifndef HAVE_SRANDOMDEV
-
-#include <sys/time.h>
-
-#include <fcntl.h>
-#include <stdlib.h>
-#include <time.h>
-#include <unistd.h>
-
-#include "compat/srandomdev.h"
-
-static int
-trydev(const char *fn, unsigned long *seed)
-{
-	int fd;
-	ssize_t sz;
-
-	fd = open(fn, O_RDONLY);
-	if (fd < 0)
-		return (-1);
-	sz = read(fd, seed, sizeof *seed);
-	(void)close(fd);
-	if (sz != sizeof *seed)
-		return (-1);
-	return (0);
-}
-
-void
-srandomdev(void)
-{
-	struct timeval tv;
-	unsigned long seed;
-
-	if (trydev("/dev/urandom", &seed)) {
-		if (trydev("/dev/random", &seed)) {
-			gettimeofday(&tv, NULL);
-			seed = (getpid() << 16) ^ tv.tv_sec ^ tv.tv_usec;
-		}
-	}
-	srandom(seed);
-}
-#endif



From phk at FreeBSD.org  Fri Dec 20 12:19:32 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 13:19:32 +0100
Subject: [master] 13c4ae1 White space nit
Message-ID: <E1Vtz3E-0004FL-Tr@project.varnish-software.com>


commit 13c4ae1cd785d2ec3559afcd08f9419ec4e88dce
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 12:19:26 2013 +0000

    White space nit

diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index a0084d5..ede05b2 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -85,8 +85,8 @@ client c1 {
 	expect resp.status == 206
 	expect resp.bodylen == 100
 
-	txreq -hdr "Range: bytes=-101" 
-	rxresp 
-	expect resp.status == 206 
-	expect resp.bodylen == 100 
+	txreq -hdr "Range: bytes=-101"
+	rxresp
+	expect resp.status == 206
+	expect resp.bodylen == 100
 } -run



From phk at FreeBSD.org  Fri Dec 20 12:24:38 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 13:24:38 +0100
Subject: [master] fc53f0d Add <stdint.h> for <vsha256.h>
Message-ID: <E1Vtz8A-00054F-Mq@project.varnish-software.com>


commit fc53f0d27a7d5d0ed6eb5bef9910a1410a7bc1f2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 12:24:23 2013 +0000

    Add <stdint.h> for <vsha256.h>

diff --git a/lib/libvarnish/vrnd.c b/lib/libvarnish/vrnd.c
index 655cf7e..aa9e477 100644
--- a/lib/libvarnish/vrnd.c
+++ b/lib/libvarnish/vrnd.c
@@ -31,6 +31,7 @@
 
 #include <errno.h>
 #include <fcntl.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>



From phk at FreeBSD.org  Fri Dec 20 12:38:47 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 13:38:47 +0100
Subject: [master] 278964f Check fchown() return values
Message-ID: <E1VtzLr-0006Ze-Sn@project.varnish-software.com>


commit 278964fee7e38ff6b33c3c87183b10798d9e9b22
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 12:37:15 2013 +0000

    Check fchown() return values

diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 507eb84..130b48f 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -123,7 +123,8 @@ vsm_n_check(void)
 			 */
 			if (pread(fd, &vsmh, sizeof vsmh, 0) == sizeof vsmh) {
 				vsmh.alloc_seq = 0;
-				(void)pwrite(fd, &vsmh, sizeof vsmh, 0);
+				assert(sizeof vsmh ==
+				    pwrite(fd, &vsmh, sizeof vsmh, 0));
 			}
 			retval = 0;
 		} else {



From phk at FreeBSD.org  Fri Dec 20 12:38:47 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 13:38:47 +0100
Subject: [master] c04662a Check fchown() return values
Message-ID: <E1VtzLr-0006Zk-UT@project.varnish-software.com>


commit c04662aea3d9e8f6c1398db9ff3df57df6f345c4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 12:38:20 2013 +0000

    Check fchown() return values

diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index d3e81de..bc7b4fe 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -244,7 +244,7 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
 		VSB_printf(sb, "Failed to create %s: %s", sf, strerror(errno));
 		return (NULL);
 	}
-	(void)fchown(sfd, mgt_param.uid, mgt_param.gid);
+	AZ(fchown(sfd, mgt_param.uid, mgt_param.gid));
 	AZ(close(sfd));
 
 
@@ -279,7 +279,7 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
 		(void)unlink(sf);
 		return (NULL);
 	}
-	(void)fchown(i, mgt_param.uid, mgt_param.gid);
+	AZ(fchown(i, mgt_param.uid, mgt_param.gid));
 	AZ(close(i));
 
 	/* Build the C-compiler command line */



From phk at FreeBSD.org  Fri Dec 20 12:38:47 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 13:38:47 +0100
Subject: [master] a04892d A bit of work on the alternate build scheme
Message-ID: <E1VtzLr-0006Zt-W8@project.varnish-software.com>


commit a04892d286eb7459c7ac0a4cf59d0c785bbcb7a6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 12:38:34 2013 +0000

    A bit of work on the alternate build scheme

diff --git a/Makefile.inc.phk b/Makefile.inc.phk
index 24756e0..72fb6b8 100644
--- a/Makefile.inc.phk
+++ b/Makefile.inc.phk
@@ -33,7 +33,7 @@ LIB_VGZ = -L $(TOPDIR)/lib/libvgz -lvgz
 LIB_PCRE = -L /usr/local/lib -lpcre
 LIB_EXECINFO = -lexecinfo
 LIB_CURSES = -lcurses
-LIB_READLINE = -lreadline
+LIB_READLINE = ${LINEDISC}
 
 #######################################################################
 ifdef USE_THREADS
diff --git a/config.phk b/config.phk
index 89ed5f9..7870b02 100644
--- a/config.phk
+++ b/config.phk
@@ -75,6 +75,13 @@ if true ; then
 		if [ -f /usr/include/$i ] ; then
 			n=`echo $i | tr '[a-z/.]' '[A-Z__]'`
 			echo "#define HAVE_$n 1" >> ${CONFIG_H}_
+		else
+			echo "#include <$i>" > _.c
+			if cc -E _.c > /dev/null 2>& 1 ; then
+				n=`echo $i | tr '[a-z/.]' '[A-Z__]'`
+				echo "#define HAVE_$n 1" >> ${CONFIG_H}_
+			fi
+			rm -f _.c
 		fi
 	done
 	echo "#define PACKAGE_TARNAME \"${ADM_PROJECT}\"" >> ${CONFIG_H}_
@@ -147,7 +154,6 @@ ask_compiler CF_CWFLAGS '
 		"-W"
 		"-fstack-protector"
 		"-Wno-format-y2k"
-		"-Wno-unused-parameter"
 		"-Wstrict-prototypes"
 		"-Wmissing-prototypes"
 		"-Wpointer-arith"
@@ -171,6 +177,15 @@ ask_compiler CF_CWFLAGS '
 			"-Wno-string-plus-int"
 		#endif
 	#endif
+	/*
+	 * Write is marked with this on some Linux, and while that is
+	 * well intentioned, they have implemented it so that the usual
+	 *	(void)bla;
+	 * marker of intent does not work.
+	 * I dont want to write bogo-code just to slip the best effort
+	 * 4xx responses in cache_http1_fetch.c past the compiler.
+	 */
+	"-Wno-unused-result"
 	'
 
 # Configuration options
@@ -191,6 +206,10 @@ ask_compiler CF_CFLAGS '
 		"-DSO_RCVTIMEO_WORKS"
 		"-DHAVE_TCP_KEEP"
 
+	#if defined(__linux__)
+		"-D_GNU_SOURCE=1"
+	#endif
+
 	"-DVCC_CC=~\~exec $(CC) -D_THREAD_SAFE -std=gnu99 -g -O2 -Wall -Werror -pthread -fpic -shared -Wl,-x -o %o %s\~~"
 
 	"-DVCC_WARNS=~\~$(CF_CWFLAGS)\~~"
@@ -219,7 +238,23 @@ if echo | rst2html > /dev/null 2>&1 ; then
 	echo "HAVE_RST2HTML=1" >> _.cache
 fi
 
+#######################################################################
+# Check for libedit
+
+if [ -f /usr/include/edit/readline/readline.h ] ; then
+	echo "LINEDISC=-ledit" >> _.cache
+elif [ -f /usr/include/editline/readline.h ] ; then
+	echo "LINEDISC=-ledit" >> _.cache
+elif [ -f /usr/include/readline/readline.h ] ; then
+	echo "LINEDISC=-lreadline" >> _.cache
+else
+	echo "LINEDISC=" >> _.cache
+fi
+
+#######################################################################
+# Done...
 cat _.cache
+
 exit 0
 
 



From phk at FreeBSD.org  Fri Dec 20 12:41:19 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 13:41:19 +0100
Subject: [master] f2462d1 Oops, forgot we could be running as non-root there.
Message-ID: <E1VtzOJ-0006rG-Ij@project.varnish-software.com>


commit f2462d139f7709bc3b678df0bcf124b8401fed1a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 12:41:07 2013 +0000

    Oops, forgot we could be running as non-root there.

diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index bc7b4fe..d3e81de 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -244,7 +244,7 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
 		VSB_printf(sb, "Failed to create %s: %s", sf, strerror(errno));
 		return (NULL);
 	}
-	AZ(fchown(sfd, mgt_param.uid, mgt_param.gid));
+	(void)fchown(sfd, mgt_param.uid, mgt_param.gid);
 	AZ(close(sfd));
 
 
@@ -279,7 +279,7 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
 		(void)unlink(sf);
 		return (NULL);
 	}
-	AZ(fchown(i, mgt_param.uid, mgt_param.gid));
+	(void)fchown(i, mgt_param.uid, mgt_param.gid);
 	AZ(close(i));
 
 	/* Build the C-compiler command line */



From phk at FreeBSD.org  Fri Dec 20 13:54:43 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 14:54:43 +0100
Subject: [master] 943777f More alt config stuff
Message-ID: <E1Vu0XL-0004qx-Mz@project.varnish-software.com>


commit 943777ffcc4ab1e465bd4c0faffb168013880fda
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 13:54:33 2013 +0000

    More alt config stuff

diff --git a/Makefile.inc.phk b/Makefile.inc.phk
index 72fb6b8..048430d 100644
--- a/Makefile.inc.phk
+++ b/Makefile.inc.phk
@@ -31,7 +31,7 @@ LIB_VARNISHTOOLS = -L $(TOPDIR)/lib/libvarnishtools -lvarnishtools
 LIB_VCC = -L $(TOPDIR)/lib/libvcc -lvcc
 LIB_VGZ = -L $(TOPDIR)/lib/libvgz -lvgz
 LIB_PCRE = -L /usr/local/lib -lpcre
-LIB_EXECINFO = -lexecinfo
+LIB_EXECINFO = ${EXECINFO}
 LIB_CURSES = -lcurses
 LIB_READLINE = ${LINEDISC}
 
diff --git a/config.phk b/config.phk
index 7870b02..3ce71ff 100644
--- a/config.phk
+++ b/config.phk
@@ -252,6 +252,13 @@ else
 fi
 
 #######################################################################
+# Check for libexecinfo (FreeBSD::backtrace()
+
+if [ -f /usr/lib/libexecinfo.so ] ; then
+	echo "EXECINFO=-lexecinfo" >> _.cache
+fi
+
+#######################################################################
 # Done...
 cat _.cache
 



From phk at FreeBSD.org  Fri Dec 20 14:12:08 2013
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Dec 2013 15:12:08 +0100
Subject: [master] 82208b2 Also cfg-check for -ldl for linux
Message-ID: <E1Vu0oC-0007Vz-Nc@project.varnish-software.com>


commit 82208b2482fa6dfce911f0e3c348e5b0f350b2fd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 20 14:11:50 2013 +0000

    Also cfg-check for -ldl for linux

diff --git a/Makefile.inc.phk b/Makefile.inc.phk
index 048430d..5c25d59 100644
--- a/Makefile.inc.phk
+++ b/Makefile.inc.phk
@@ -31,9 +31,9 @@ LIB_VARNISHTOOLS = -L $(TOPDIR)/lib/libvarnishtools -lvarnishtools
 LIB_VCC = -L $(TOPDIR)/lib/libvcc -lvcc
 LIB_VGZ = -L $(TOPDIR)/lib/libvgz -lvgz
 LIB_PCRE = -L /usr/local/lib -lpcre
-LIB_EXECINFO = ${EXECINFO}
+LIB_EXECINFO = ${CFG_EXECINFO}
 LIB_CURSES = -lcurses
-LIB_READLINE = ${LINEDISC}
+LIB_READLINE = ${CFG_LINEDISC}
 
 #######################################################################
 ifdef USE_THREADS
@@ -43,6 +43,7 @@ endif
 
 ifdef USE_DLOPEN
 PROG_LDFLAGS += -Wl,-E
+PROG_LIBS += ${CFG_LIBDL}
 endif
 
 #######################################################################
@@ -111,7 +112,7 @@ include _.depprog
 endif
 
 $(PROGNAME):	$(PROG_OBJ)
-	$(CC) $(PROG_LDFLAGS) -o $(PROGNAME) $^ $(LD_ADD)
+	$(CC) $(PROG_LDFLAGS) -o $(PROGNAME) $^ $(PROG_LIBS) $(LD_ADD)
 
 .PHONY: prog_install
 prog_install:	$(PROGNAME)
diff --git a/config.phk b/config.phk
index 3ce71ff..aa6856f 100644
--- a/config.phk
+++ b/config.phk
@@ -242,20 +242,27 @@ fi
 # Check for libedit
 
 if [ -f /usr/include/edit/readline/readline.h ] ; then
-	echo "LINEDISC=-ledit" >> _.cache
+	echo "CFG_LINEDISC=-ledit" >> _.cache
 elif [ -f /usr/include/editline/readline.h ] ; then
-	echo "LINEDISC=-ledit" >> _.cache
+	echo "CFG_LINEDISC=-ledit" >> _.cache
 elif [ -f /usr/include/readline/readline.h ] ; then
-	echo "LINEDISC=-lreadline" >> _.cache
+	echo "CFG_LINEDISC=-lreadline" >> _.cache
 else
-	echo "LINEDISC=" >> _.cache
+	echo "CFG_LINEDISC=" >> _.cache
 fi
 
 #######################################################################
-# Check for libexecinfo (FreeBSD::backtrace()
+# Check for libexecinfo for FreeBSD::backtrace()
 
 if [ -f /usr/lib/libexecinfo.so ] ; then
-	echo "EXECINFO=-lexecinfo" >> _.cache
+	echo "CFG_EXECINFO=-lexecinfo" >> _.cache
+fi
+
+#######################################################################
+# Check for libdl for Linux::dlopen()
+
+if [ `uname -s` = "Linux" ] ; then
+	echo "CFG_LIBDL=-ldl" >> _.cache
 fi
 
 #######################################################################